testing and plotting iteration performance

doc/plot_iteration_performance.py

@@ -0,0 +1,45 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+from collections import defaultdict
+import re
+
+data = json.load(open("doc/iteration.perf"))
+
+bench = defaultdict(lambda:[])
+for x in data["benchmarks"]:
+    if x["run_type"] != "aggregate":
+        continue
+    if x["aggregate_name"] != "mean":
+        continue
+    name, arg = x["run_name"].split("/")
+    m = re.match("(\S+)<(\S+), *(\S+)>", name)
+    name = m.group(1)
+    hist = m.group(2)
+    extra = m.group(3)
+    bench[(name, hist, extra)].append((int(arg) ** 3, x["cpu_time"]))
+
+plt.figure(figsize=(7, 6))
+handles = []
+for (name, axis, extra), v in bench.items():
+    v = np.sort(v).T
+    # if "semi_dynamic" in axis: continue
+    if "LessNaive" in name: continue
+    if extra == "false": continue
+    lw = 3 if "Indexed" in name else 1.5
+    col = {"NaiveForLoop": "r", "InsiderForLoop": "C0", "IndexedLoop": "k"}.get(name, "k")
+    ls = {"static_tag": "-", "semi_dynamic_tag": "--", "full_dynamic_tag": ":"}[axis]
+    name2 = {"NaiveForLoop": "nested for (naive)", "InsiderForLoop" : "nested for (opt.)", "IndexedLoop": "indexed"}.get(name, name)
+    axis2 = {"static_tag": "tuple", "semi_dynamic_tag": "vector",
+             "full_dynamic_tag": "vector of variant"}.get(axis, axis)
+    h = plt.plot(v[0], v[1], lw=lw, ls=ls, color=col,
+                 label=r"%s: ${\mathit{axes}}$ = %s" % (name2, axis2))[0]
+    handles.append(h)
+handles.sort(key=lambda x: x.get_label())
+plt.loglog()
+plt.legend(handles=handles, fontsize="xx-small")
+plt.ylabel("CPU time (less is better)")
+plt.xlabel("number of bins in 3D histogram")
+plt.tight_layout()
+plt.savefig("iteration_performance.svg")
+plt.show()

test/iteration_bench.cpp

@@ -12,29 +12,29 @@ struct static_tag {};
 struct semi_dynamic_tag {};
 struct full_dynamic_tag {};
 
-auto make_histogram(static_tag) {
+auto make_histogram(static_tag, unsigned n) {
   using namespace boost::histogram;
-  return make_histogram_with(std::vector<unsigned>(), axis::integer<>(0, 20),
+  return make_histogram_with(std::vector<unsigned>(), axis::integer<>(0, n),
-                             axis::integer<>(0, 20), axis::integer<>(0, 20));
+                             axis::integer<>(0, n), axis::integer<>(0, n));
 }
 
-auto make_histogram(semi_dynamic_tag) {
+auto make_histogram(semi_dynamic_tag, unsigned n) {
   using namespace boost::histogram;
-  std::vector<axis::integer<>> axes = {axis::integer<>(0, 20), axis::integer<>(0, 20),
+  std::vector<axis::integer<>> axes = {axis::integer<>(0, n), axis::integer<>(0, n),
-                                       axis::integer<>(0, 20)};
+                                       axis::integer<>(0, n)};
   return make_histogram_with(std::vector<unsigned>(), axes);
 }
 
-auto make_histogram(full_dynamic_tag) {
+auto make_histogram(full_dynamic_tag, unsigned n) {
   using namespace boost::histogram;
   std::vector<axis::variant<axis::integer<>>> axes = {
-      axis::integer<>(0, 20), axis::integer<>(0, 20), axis::integer<>(0, 20)};
+      axis::integer<>(0, n), axis::integer<>(0, n), axis::integer<>(0, n)};
   return make_histogram_with(std::vector<unsigned>(), axes);
 }
 
 template <class Tag, bool include_extra_bins>
 static void NaiveForLoop(benchmark::State& state) {
-  auto h = make_histogram(Tag());
+  auto h = make_histogram(Tag(), state.range(0));
   for (auto _ : state) {
     for (int i = -include_extra_bins; i < h.axis(0).size() + include_extra_bins; ++i)
       for (int j = -include_extra_bins; j < h.axis(1).size() + include_extra_bins; ++j)
@@ -43,22 +43,10 @@ static void NaiveForLoop(benchmark::State& state) {
   }
 }
 
-template <class Tag, bool include_extra_bins>
-static void LessNaiveForLoop(benchmark::State& state) {
-  static_assert(!include_extra_bins, "cannot include extra bins here");
-  using namespace boost::histogram::literals;
-  auto h = make_histogram(Tag());
-  for (auto _ : state) {
-    for (auto i : h.axis(0_c))
-      for (auto j : h.axis(1_c))
-        for (auto k : h.axis(2_c)) benchmark::DoNotOptimize(h.at(i, j, k));
-  }
-}
-
 template <class Tag, bool include_extra_bins>
 static void InsiderForLoop(benchmark::State& state) {
   using namespace boost::histogram::literals;
-  auto h = make_histogram(Tag());
+  auto h = make_histogram(Tag(), state.range(0));
   for (auto _ : state) {
     for (int k = -include_extra_bins, nk = h.axis(2_c).size() + include_extra_bins;
          k < nk; ++k)
@@ -72,7 +60,7 @@ static void InsiderForLoop(benchmark::State& state) {
 
 template <class Tag, bool include_extra_bins>
 static void IndexedLoop(benchmark::State& state) {
-  auto h = make_histogram(Tag());
+  auto h = make_histogram(Tag(), state.range(0));
   for (auto _ : state) {
     for (auto x : boost::histogram::indexed(h, include_extra_bins)) {
       benchmark::DoNotOptimize(*x);
@@ -83,29 +71,50 @@ static void IndexedLoop(benchmark::State& state) {
   }
 }
 
-BENCHMARK_TEMPLATE(NaiveForLoop, static_tag, false);
+BENCHMARK_TEMPLATE(NaiveForLoop, static_tag, false)->RangeMultiplier(2)->Range(4, 128);
-BENCHMARK_TEMPLATE(LessNaiveForLoop, static_tag, false);
+BENCHMARK_TEMPLATE(InsiderForLoop, static_tag, false)->RangeMultiplier(2)->Range(4, 128);
-BENCHMARK_TEMPLATE(InsiderForLoop, static_tag, false);
+BENCHMARK_TEMPLATE(IndexedLoop, static_tag, false)->RangeMultiplier(2)->Range(4, 128);
-BENCHMARK_TEMPLATE(IndexedLoop, static_tag, false);
 
-BENCHMARK_TEMPLATE(NaiveForLoop, static_tag, true);
+BENCHMARK_TEMPLATE(NaiveForLoop, static_tag, true)->RangeMultiplier(2)->Range(4, 128);
-BENCHMARK_TEMPLATE(InsiderForLoop, static_tag, true);
+BENCHMARK_TEMPLATE(InsiderForLoop, static_tag, true)->RangeMultiplier(2)->Range(4, 128);
-BENCHMARK_TEMPLATE(IndexedLoop, static_tag, true);
+BENCHMARK_TEMPLATE(IndexedLoop, static_tag, true)->RangeMultiplier(2)->Range(4, 128);
 
-BENCHMARK_TEMPLATE(NaiveForLoop, semi_dynamic_tag, false);
+BENCHMARK_TEMPLATE(NaiveForLoop, semi_dynamic_tag, false)
-BENCHMARK_TEMPLATE(LessNaiveForLoop, semi_dynamic_tag, false);
+    ->RangeMultiplier(2)
-BENCHMARK_TEMPLATE(InsiderForLoop, semi_dynamic_tag, false);
+    ->Range(4, 128);
-BENCHMARK_TEMPLATE(IndexedLoop, semi_dynamic_tag, false);
+BENCHMARK_TEMPLATE(InsiderForLoop, semi_dynamic_tag, false)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);
+BENCHMARK_TEMPLATE(IndexedLoop, semi_dynamic_tag, false)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);
 
-BENCHMARK_TEMPLATE(NaiveForLoop, semi_dynamic_tag, true);
+BENCHMARK_TEMPLATE(NaiveForLoop, semi_dynamic_tag, true)
-BENCHMARK_TEMPLATE(InsiderForLoop, semi_dynamic_tag, true);
+    ->RangeMultiplier(2)
-BENCHMARK_TEMPLATE(IndexedLoop, semi_dynamic_tag, true);
+    ->Range(4, 128);
+BENCHMARK_TEMPLATE(InsiderForLoop, semi_dynamic_tag, true)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);
+BENCHMARK_TEMPLATE(IndexedLoop, semi_dynamic_tag, true)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);
 
-BENCHMARK_TEMPLATE(NaiveForLoop, full_dynamic_tag, false);
+BENCHMARK_TEMPLATE(NaiveForLoop, full_dynamic_tag, false)
-BENCHMARK_TEMPLATE(LessNaiveForLoop, full_dynamic_tag, false);
+    ->RangeMultiplier(2)
-BENCHMARK_TEMPLATE(InsiderForLoop, full_dynamic_tag, false);
+    ->Range(4, 128);
-BENCHMARK_TEMPLATE(IndexedLoop, full_dynamic_tag, false);
+BENCHMARK_TEMPLATE(InsiderForLoop, full_dynamic_tag, false)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);
+BENCHMARK_TEMPLATE(IndexedLoop, full_dynamic_tag, false)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);
 
-BENCHMARK_TEMPLATE(NaiveForLoop, full_dynamic_tag, true);
+BENCHMARK_TEMPLATE(NaiveForLoop, full_dynamic_tag, true)
-BENCHMARK_TEMPLATE(InsiderForLoop, full_dynamic_tag, true);
+    ->RangeMultiplier(2)
-BENCHMARK_TEMPLATE(IndexedLoop, full_dynamic_tag, true);
+    ->Range(4, 128);
+BENCHMARK_TEMPLATE(InsiderForLoop, full_dynamic_tag, true)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);
+BENCHMARK_TEMPLATE(IndexedLoop, full_dynamic_tag, true)
+    ->RangeMultiplier(2)
+    ->Range(4, 128);