simpler and faster benchmarks

benchmark/plot_benchmarks.py

@@ -24,7 +24,7 @@ def get_benchmarks(results):
                 benchmarks[name].append((commits.index(hash), time))
     return benchmarks
 
-with shelve.open(sys.argv[1]) as results:
+with shelve.open("benchmark_results") as results:
     benchmarks = get_benchmarks(results)
 
 fig, ax = plt.subplots(4, 1, figsize=(10, 10), sharex=True)
@@ -80,7 +80,7 @@ def onpick(event):
 
     def worker(fig, ax, hash, lock):
         with lock:
-            with shelve.open(sys.argv[1]) as results:
+            with shelve.open("benchmark_results") as results:
                 del results[hash]
                 run(results, comments, hash)
                 benchmarks = get_benchmarks(results)

benchmark/run_benchmarks.py

@@ -8,7 +8,7 @@ Run this from a special build directory that uses the benchmark folder as root
 
 This creates a database, benchmark_results. Plot it:
 
-    ../plot_benchmarks.py benchmark_results
+    ../plot_benchmarks.py
 
 The script leaves the include folder in a modified state. To clean up, do:
 
@@ -21,7 +21,7 @@ import tempfile
 import os
 import shelve
 import json
-import sys
+import argparse
 
 
 def get_commits():
@@ -44,8 +44,8 @@ def recursion(results, commits, comments, ia, ib):
     run(results, comments, commits[ic])
     if all([results[commits[i]] is None for i in (ia, ib, ic)]):
         return
-    recursion(results, commits, comments, ia, ic)
     recursion(results, commits, comments, ic, ib)
+    recursion(results, commits, comments, ia, ic)
 
 
 def run(results, comments, hash):
@@ -65,35 +65,31 @@ def run(results, comments, hash):
             sys.stderr.write(out.read().decode("utf-8") + "\n")
             return
     print(hash, "run")
-    s = subp.check_output(("./histogram_filling", "--benchmark_format=json"))
+    s = subp.check_output(("./histogram_filling", "--benchmark_format=json", "--benchmark_filter=normal"))
     results[hash] = d = json.loads(s)
     for benchmark in d["benchmarks"]:
         print(benchmark["name"], min(benchmark["real_time"], benchmark["cpu_time"]))
 
 
 def main():
+    commits, comments = get_commits()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("first", type=str, default=commits[0])
+    parser.add_argument("last", type=str, nargs="?", default=commits[-1])
+    parser.add_argument("-f", action="store_true")
+
+    args = parser.parse_args()
+
     with shelve.open("benchmark_results") as results:
-        commits, comments = get_commits()
+        a = commits.index(args.first)
-        if len(sys.argv) == 2:
+        b = commits.index(args.last)
-            # redo this commit
+        if args.f:
-            hash = sys.argv[1]
+            for hash in commits[a:b+1]:
-            del results[hash]
+                del results[hash]
-            run(results, comments, hash)
+        run(results, comments, args.first)
-        else:
+        run(results, comments, args.last)
-            if len(sys.argv) == 3:
+        recursion(results, commits, comments, a, b)
-                first = sys.argv[1]
-                last = sys.argv[2]
-            else:
-                first = commits[0]
-                last = commits[-1]
-            # retry first, last if previous builds failed
-            if first in results and results[first] is None:
-                del results[first]
-            if last in results and results[last] is None:
-                del results[last]
-            run(results, comments, first)
-            run(results, comments, last)
-            recursion(results, commits, comments, commits.index(first), commits.index(last))
 
 if __name__ == "__main__":
     main()