analysis: test the frozen depth-retention margin

4 files changed, 101 insertions, 1 deletions

diff --git a/RESULTS.md b/RESULTS.md
index 29c3e33..7d60c4f 100644
--- a/RESULTS.md
+++ b/RESULTS.md
@@ -88,6 +88,13 @@ depth grows 12x and wall time grows 9.87x (geometric mean). Its paired gap to BP
 3.69 and 4.35 points at every tested depth. Flattened CIFAR itself is depth-flat, so this establishes
 preservation under depth, not a claim that extra depth improves the task.
 
+The `-1`-point retention margin was fixed in `experiments/finalize_claims.sh` before the missing
+depth cells were generated. A paired one-sided t-test rejects the null that SDIL's mean d5-to-d60
+change is at most `-1` point (`t(4)=5.030`, `p=0.0037`; one-sided 95% lower bound `-0.547`). The
+same complete analysis rejects that null for BP (`p=0.0359`) and FA (`p=0.0321`) but not DFA
+(`p=0.2310`; lower bound `-1.211`). This is an equivalence-to-the-prespecified-margin statement,
+not a post-hoc test that the depth change is exactly zero.
+
 The mean-based nondominated frontier among local methods is DFA-d5 (`5.01 s`, `37.77%`), FA-d5
 (`5.72 s`, `38.92%`), then SDIL-d5 (`9.84 s`, `41.46%`). BP is shown only as a nonlocal reference.
 Thus SDIL occupies the high-accuracy end of the audited local-learning frontier; the deeper SDIL
diff --git a/experiments/plot_main_figures.py b/experiments/plot_main_figures.py
index 69009ab..a614343 100644
--- a/experiments/plot_main_figures.py
+++ b/experiments/plot_main_figures.py
@@ -17,6 +17,7 @@ import matplotlib
 matplotlib.use("Agg")
 import matplotlib.pyplot as plt
 from matplotlib.ticker import FuncFormatter
+from scipy.stats import t as student_t
 
 
 COLORS = {
@@ -410,6 +411,36 @@ def audited_statistics(root):
                     statistics.mean(math.log(value) for value in wall_ratios)),
                 "d60_over_d5_wall_ratios": wall_ratios,
             }
+            if len(accuracy_changes) > 1:
+                mean_change, sd_change = mean_sd(accuracy_changes)
+                standard_error = sd_change / math.sqrt(len(accuracy_changes))
+                margin = -1.0
+                if standard_error > 0:
+                    statistic = (mean_change - margin) / standard_error
+                    one_sided_p = float(student_t.sf(
+                        statistic, df=len(accuracy_changes) - 1))
+                    one_sided_lower = mean_change - float(student_t.ppf(
+                        0.95, df=len(accuracy_changes) - 1)) * standard_error
+                    two_sided_radius = float(student_t.ppf(
+                        0.975, df=len(accuracy_changes) - 1)) * standard_error
+                else:
+                    statistic = math.inf if mean_change > margin else -math.inf
+                    one_sided_p = 0.0 if mean_change > margin else 1.0
+                    one_sided_lower = mean_change
+                    two_sided_radius = 0.0
+                endpoint["paired_accuracy_retention_test"] = {
+                    "null": "mean d60-minus-d5 accuracy change <= -1 point",
+                    "alternative": "mean d60-minus-d5 accuracy change > -1 point",
+                    "predeclared_margin_points": margin,
+                    "degrees_of_freedom": len(accuracy_changes) - 1,
+                    "t_statistic": statistic,
+                    "one_sided_p": one_sided_p,
+                    "one_sided_95_lower_bound_points": one_sided_lower,
+                    "two_sided_95_interval_points": [
+                        mean_change - two_sided_radius,
+                        mean_change + two_sided_radius],
+                    "reject_drop_at_least_1pt_alpha_0p05": one_sided_p < 0.05,
+                }
             if name != "BP":
                 alignment_changes = [
                     early_alignment(deep[s]) - early_alignment(shallow[s]) for s in seeds
@@ -810,6 +841,9 @@ depth and initialization seed. **c,** Mean per-sample cosine between the teachin
 descent direction, averaged over the earliest third of hidden layers. Exact gradients are used only
 for this diagnostic, never for local learning. “Scaling” here means preserving useful accuracy and
 credit assignment as depth grows; flattened-CIFAR accuracy itself does not increase with depth.
+Against the predeclared `-1`-point retention margin, the paired one-sided depth-5-to-60 test rejects
+a drop of at least one point for SDIL (`p=0.0037`) but not DFA (`p=0.2310`); results for every
+method are retained in the figure manifest.
 
 **Figure 3 | Somato-dendritic innovation is necessary under predictable apical traffic.** Mean ±
 sample standard deviation across five seeds on MNIST, using depth-3, width-256 networks for 15
diff --git a/results/figs/main_figure_captions.md b/results/figs/main_figure_captions.md
index 03c16c8..6b8562e 100644
--- a/results/figs/main_figure_captions.md
+++ b/results/figs/main_figure_captions.md
@@ -22,6 +22,9 @@ depth and initialization seed. **c,** Mean per-sample cosine between the teachin
 descent direction, averaged over the earliest third of hidden layers. Exact gradients are used only
 for this diagnostic, never for local learning. “Scaling” here means preserving useful accuracy and
 credit assignment as depth grows; flattened-CIFAR accuracy itself does not increase with depth.
+Against the predeclared `-1`-point retention margin, the paired one-sided depth-5-to-60 test rejects
+a drop of at least one point for SDIL (`p=0.0037`) but not DFA (`p=0.2310`); results for every
+method are retained in the figure manifest.
 
 **Figure 3 | Somato-dendritic innovation is necessary under predictable apical traffic.** Mean ±
 sample standard deviation across five seeds on MNIST, using depth-3, width-256 networks for 15
diff --git a/results/figs/main_figure_manifest.json b/results/figs/main_figure_manifest.json
index 294cf94..46527d3 100644
--- a/results/figs/main_figure_manifest.json
+++ b/results/figs/main_figure_manifest.json
@@ -2373,7 +2373,21 @@
           8.59384509799933,
           8.346117401221937,
           8.787989553791133
-        ]
+        ],
+        "paired_accuracy_retention_test": {
+          "null": "mean d60-minus-d5 accuracy change <= -1 point",
+          "alternative": "mean d60-minus-d5 accuracy change > -1 point",
+          "predeclared_margin_points": -1.0,
+          "degrees_of_freedom": 4,
+          "t_statistic": 2.4311488030619435,
+          "one_sided_p": 0.03594586798395181,
+          "one_sided_95_lower_bound_points": -0.8672859622437409,
+          "two_sided_95_interval_points": [
+            -1.1531084453710267,
+            1.3091084453710273
+          ],
+          "reject_drop_at_least_1pt_alpha_0p05": true
+        }
       },
       "FA": {
         "seeds": [
@@ -2393,6 +2407,20 @@
           6.7033463045694015,
           9.101054953876158
         ],
+        "paired_accuracy_retention_test": {
+          "null": "mean d60-minus-d5 accuracy change <= -1 point",
+          "alternative": "mean d60-minus-d5 accuracy change > -1 point",
+          "predeclared_margin_points": -1.0,
+          "degrees_of_freedom": 4,
+          "t_statistic": 2.5357066333490828,
+          "one_sided_p": 0.03213672421335003,
+          "one_sided_95_lower_bound_points": -0.9197283470326103,
+          "two_sided_95_interval_points": [
+            -1.0478494586936888,
+            0.05584945869368485
+          ],
+          "reject_drop_at_least_1pt_alpha_0p05": true
+        },
         "d60_minus_d5_early_alignment_mean": -0.052620660066604596,
         "d60_minus_d5_early_alignment_sd": 0.013629272999240246
       },
@@ -2414,6 +2442,20 @@
           9.083443515320905,
           8.984656533045191
         ],
+        "paired_accuracy_retention_test": {
+          "null": "mean d60-minus-d5 accuracy change <= -1 point",
+          "alternative": "mean d60-minus-d5 accuracy change > -1 point",
+          "predeclared_margin_points": -1.0,
+          "degrees_of_freedom": 4,
+          "t_statistic": 0.8126587379130075,
+          "one_sided_p": 0.2310059494410654,
+          "one_sided_95_lower_bound_points": -1.211028859093041,
+          "two_sided_95_interval_points": [
+            -1.3141444444474168,
+            -0.4258555555525847
+          ],
+          "reject_drop_at_least_1pt_alpha_0p05": false
+        },
         "d60_minus_d5_early_alignment_mean": -0.46736509441630913,
         "d60_minus_d5_early_alignment_sd": 0.03641498460995918
       },
@@ -2435,6 +2477,20 @@
           9.78526636436608,
           10.016996559591552
         ],
+        "paired_accuracy_retention_test": {
+          "null": "mean d60-minus-d5 accuracy change <= -1 point",
+          "alternative": "mean d60-minus-d5 accuracy change > -1 point",
+          "predeclared_margin_points": -1.0,
+          "degrees_of_freedom": 4,
+          "t_statistic": 5.030200405607772,
+          "one_sided_p": 0.003666228038780643,
+          "one_sided_95_lower_bound_points": -0.5471142775513921,
+          "two_sided_95_interval_points": [
+            -0.6478367613052975,
+            0.21983676130529367
+          ],
+          "reject_drop_at_least_1pt_alpha_0p05": true
+        },
         "d60_minus_d5_early_alignment_mean": -0.46029999762773516,
         "d60_minus_d5_early_alignment_sd": 0.02065602739281337
       }