figures: export auditable claim statistics

1 files changed, 78 insertions, 0 deletions

diff --git a/experiments/plot_main_figures.py b/experiments/plot_main_figures.py
index e171181..23b4cc9 100644
--- a/experiments/plot_main_figures.py
+++ b/experiments/plot_main_figures.py
@@ -189,6 +189,83 @@ def nondominated(points):
     return front
 
 
+def cell_summary(rows, include_alignment=False):
+    if not rows:
+        return None
+    acc = [100 * r["final"]["test_acc"] for r in rows]
+    wall = [r["final"]["wall_s"] for r in rows]
+    acc_mean, acc_sd = mean_sd(acc)
+    wall_mean, wall_sd = mean_sd(wall)
+    summary = {
+        "n": len(rows),
+        "seeds": sorted(r["args"]["seed"] for r in rows),
+        "accuracy_percent_mean": acc_mean,
+        "accuracy_percent_sd": acc_sd,
+        "wall_s_mean": wall_mean,
+        "wall_s_sd": wall_sd,
+    }
+    if include_alignment:
+        alignments = [v for r in rows if (v := early_alignment(r)) is not None]
+        if alignments:
+            summary["early_alignment_mean"], summary["early_alignment_sd"] = mean_sd(
+                alignments)
+    return summary
+
+
+def audited_statistics(root):
+    scale = scaling_records(root)
+    sg = scaling_groups(scale)
+    scaling = {}
+    frontier_candidates = []
+    for name in ("BP", "FA", "DFA", "SDIL"):
+        for depth in DEPTHS:
+            summary = cell_summary(sg.get((name, depth), []), include_alignment=name != "BP")
+            if summary is None:
+                continue
+            scaling[f"{name}_d{depth}"] = summary
+            if name != "BP":
+                frontier_candidates.append((summary["wall_s_mean"],
+                                            summary["accuracy_percent_mean"],
+                                            name, depth))
+    front = nondominated(frontier_candidates)
+
+    _, eg = ep_groups(root)
+    ep_exact = {}
+    paired = {}
+    for name in ("BP", "DFA", "SDIL", "EP"):
+        for depth in (1, 2):
+            summary = cell_summary(eg.get((name, depth), []))
+            if summary is not None:
+                ep_exact[f"{name}_d{depth}"] = summary
+    for depth in (1, 2):
+        sdil = {r["args"]["seed"]: r for r in eg.get(("SDIL", depth), [])}
+        ep = {r["args"]["seed"]: r for r in eg.get(("EP", depth), [])}
+        seeds = sorted(set(sdil) & set(ep))
+        if seeds:
+            gaps = [100 * (sdil[s]["final"]["test_acc"] - ep[s]["final"]["test_acc"])
+                    for s in seeds]
+            speedups = [ep[s]["final"]["wall_s"] / sdil[s]["final"]["wall_s"]
+                        for s in seeds]
+            paired[f"SDIL_minus_EP_d{depth}"] = {
+                "seeds": seeds,
+                "accuracy_point_gap_mean": mean_sd(gaps)[0],
+                "accuracy_point_gap_sd": mean_sd(gaps)[1],
+                "wall_speedup_geomean": math.exp(statistics.mean(math.log(v)
+                                                                  for v in speedups)),
+                "wall_speedups": speedups,
+            }
+    return {
+        "cifar_scaling": scaling,
+        "local_pareto_frontier": [
+            {"method": p[2], "depth": p[3], "wall_s_mean": p[0],
+             "accuracy_percent_mean": p[1]} for p in front
+        ],
+        "sdil_is_high_accuracy_frontier_endpoint": bool(front and front[-1][2] == "SDIL"),
+        "mnist_exact_ep": ep_exact,
+        "paired_ep_comparison": paired,
+    }
+
+
 def setup_style():
     plt.rcParams.update({
         "font.family": "DejaVu Sans",
@@ -415,6 +492,7 @@ def main():
         },
         "missing_cells": missing1 + missing2,
         "sources": [{"path": p, "sha256": file_hash(p)} for p in source_paths],
+        "statistics": audited_statistics(args.results),
         "outputs": ["figure1_pareto.pdf", "figure1_pareto.png",
                     "figure2_scaling.pdf", "figure2_scaling.png"],
     }