analysis: test the frozen depth-retention margin

1 files changed, 34 insertions, 0 deletions

diff --git a/experiments/plot_main_figures.py b/experiments/plot_main_figures.py
index 69009ab..a614343 100644
--- a/experiments/plot_main_figures.py
+++ b/experiments/plot_main_figures.py
@@ -17,6 +17,7 @@ import matplotlib
 matplotlib.use("Agg")
 import matplotlib.pyplot as plt
 from matplotlib.ticker import FuncFormatter
+from scipy.stats import t as student_t
 
 
 COLORS = {
@@ -410,6 +411,36 @@ def audited_statistics(root):
                     statistics.mean(math.log(value) for value in wall_ratios)),
                 "d60_over_d5_wall_ratios": wall_ratios,
             }
+            if len(accuracy_changes) > 1:
+                mean_change, sd_change = mean_sd(accuracy_changes)
+                standard_error = sd_change / math.sqrt(len(accuracy_changes))
+                margin = -1.0
+                if standard_error > 0:
+                    statistic = (mean_change - margin) / standard_error
+                    one_sided_p = float(student_t.sf(
+                        statistic, df=len(accuracy_changes) - 1))
+                    one_sided_lower = mean_change - float(student_t.ppf(
+                        0.95, df=len(accuracy_changes) - 1)) * standard_error
+                    two_sided_radius = float(student_t.ppf(
+                        0.975, df=len(accuracy_changes) - 1)) * standard_error
+                else:
+                    statistic = math.inf if mean_change > margin else -math.inf
+                    one_sided_p = 0.0 if mean_change > margin else 1.0
+                    one_sided_lower = mean_change
+                    two_sided_radius = 0.0
+                endpoint["paired_accuracy_retention_test"] = {
+                    "null": "mean d60-minus-d5 accuracy change <= -1 point",
+                    "alternative": "mean d60-minus-d5 accuracy change > -1 point",
+                    "predeclared_margin_points": margin,
+                    "degrees_of_freedom": len(accuracy_changes) - 1,
+                    "t_statistic": statistic,
+                    "one_sided_p": one_sided_p,
+                    "one_sided_95_lower_bound_points": one_sided_lower,
+                    "two_sided_95_interval_points": [
+                        mean_change - two_sided_radius,
+                        mean_change + two_sided_radius],
+                    "reject_drop_at_least_1pt_alpha_0p05": one_sided_p < 0.05,
+                }
             if name != "BP":
                 alignment_changes = [
                     early_alignment(deep[s]) - early_alignment(shallow[s]) for s in seeds
@@ -810,6 +841,9 @@ depth and initialization seed. **c,** Mean per-sample cosine between the teachin
 descent direction, averaged over the earliest third of hidden layers. Exact gradients are used only
 for this diagnostic, never for local learning. “Scaling” here means preserving useful accuracy and
 credit assignment as depth grows; flattened-CIFAR accuracy itself does not increase with depth.
+Against the predeclared `-1`-point retention margin, the paired one-sided depth-5-to-60 test rejects
+a drop of at least one point for SDIL (`p=0.0037`) but not DFA (`p=0.2310`); results for every
+method are retained in the figure manifest.
 
 **Figure 3 | Somato-dendritic innovation is necessary under predictable apical traffic.** Mean ±
 sample standard deviation across five seeds on MNIST, using depth-3, width-256 networks for 15