figures: export paired scaling statistics

1 files changed, 46 insertions, 0 deletions

diff --git a/experiments/plot_main_figures.py b/experiments/plot_main_figures.py
index 9005caa..ad969dc 100644
--- a/experiments/plot_main_figures.py
+++ b/experiments/plot_main_figures.py
@@ -246,6 +246,8 @@ def audited_statistics(root):
     scale = scaling_records(root)
     sg = scaling_groups(scale)
     scaling = {}
+    paired_bp = {}
+    depth_endpoints = {}
     frontier_candidates = []
     for name in ("BP", "FA", "DFA", "SDIL"):
         for depth in DEPTHS:
@@ -257,6 +259,48 @@ def audited_statistics(root):
                 frontier_candidates.append((summary["wall_s_mean"],
                                             summary["accuracy_percent_mean"],
                                             name, depth))
+    for name in ("FA", "DFA", "SDIL"):
+        for depth in DEPTHS:
+            bp = {r["args"]["seed"]: r for r in sg.get(("BP", depth), [])}
+            local = {r["args"]["seed"]: r for r in sg.get((name, depth), [])}
+            seeds = sorted(set(bp) & set(local))
+            if seeds:
+                gaps = [100 * (bp[s]["final"]["test_acc"]
+                               - local[s]["final"]["test_acc"]) for s in seeds]
+                paired_bp[f"BP_minus_{name}_d{depth}"] = {
+                    "seeds": seeds,
+                    "accuracy_point_gap_mean": mean_sd(gaps)[0],
+                    "accuracy_point_gap_sd": mean_sd(gaps)[1],
+                }
+    for name in ("BP", "FA", "DFA", "SDIL"):
+        shallow = {r["args"]["seed"]: r for r in sg.get((name, DEPTHS[0]), [])}
+        deep = {r["args"]["seed"]: r for r in sg.get((name, DEPTHS[-1]), [])}
+        seeds = sorted(set(shallow) & set(deep))
+        if seeds:
+            accuracy_changes = [
+                100 * (deep[s]["final"]["test_acc"]
+                       - shallow[s]["final"]["test_acc"]) for s in seeds]
+            wall_ratios = [deep[s]["final"]["wall_s"] / shallow[s]["final"]["wall_s"]
+                           for s in seeds]
+            endpoint = {
+                "seeds": seeds,
+                "d60_minus_d5_accuracy_points_mean": mean_sd(accuracy_changes)[0],
+                "d60_minus_d5_accuracy_points_sd": mean_sd(accuracy_changes)[1],
+                "d60_over_d5_wall_geomean": math.exp(
+                    statistics.mean(math.log(value) for value in wall_ratios)),
+                "d60_over_d5_wall_ratios": wall_ratios,
+            }
+            if name != "BP":
+                alignment_changes = [
+                    early_alignment(deep[s]) - early_alignment(shallow[s]) for s in seeds
+                    if (early_alignment(deep[s]) is not None
+                        and early_alignment(shallow[s]) is not None)]
+                if alignment_changes:
+                    endpoint["d60_minus_d5_early_alignment_mean"] = mean_sd(
+                        alignment_changes)[0]
+                    endpoint["d60_minus_d5_early_alignment_sd"] = mean_sd(
+                        alignment_changes)[1]
+            depth_endpoints[name] = endpoint
     front = nondominated(frontier_candidates)
 
     _, eg = ep_groups(root)
@@ -286,6 +330,8 @@ def audited_statistics(root):
             }
     return {
         "cifar_scaling": scaling,
+        "paired_gap_to_bp": paired_bp,
+        "depth5_to_depth60": depth_endpoints,
         "local_pareto_frontier": [
             {"method": p[2], "depth": p[3], "wall_s_mean": p[0],
              "accuracy_percent_mean": p[1]} for p in front