From 94a90c664ff4ba60ff3cbe7327bd4a571c31f438 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 06:57:24 -0500 Subject: analysis: test the frozen depth-retention margin --- experiments/plot_main_figures.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'experiments/plot_main_figures.py') diff --git a/experiments/plot_main_figures.py b/experiments/plot_main_figures.py index 69009ab..a614343 100644 --- a/experiments/plot_main_figures.py +++ b/experiments/plot_main_figures.py @@ -17,6 +17,7 @@ import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter +from scipy.stats import t as student_t COLORS = { @@ -410,6 +411,36 @@ def audited_statistics(root): statistics.mean(math.log(value) for value in wall_ratios)), "d60_over_d5_wall_ratios": wall_ratios, } + if len(accuracy_changes) > 1: + mean_change, sd_change = mean_sd(accuracy_changes) + standard_error = sd_change / math.sqrt(len(accuracy_changes)) + margin = -1.0 + if standard_error > 0: + statistic = (mean_change - margin) / standard_error + one_sided_p = float(student_t.sf( + statistic, df=len(accuracy_changes) - 1)) + one_sided_lower = mean_change - float(student_t.ppf( + 0.95, df=len(accuracy_changes) - 1)) * standard_error + two_sided_radius = float(student_t.ppf( + 0.975, df=len(accuracy_changes) - 1)) * standard_error + else: + statistic = math.inf if mean_change > margin else -math.inf + one_sided_p = 0.0 if mean_change > margin else 1.0 + one_sided_lower = mean_change + two_sided_radius = 0.0 + endpoint["paired_accuracy_retention_test"] = { + "null": "mean d60-minus-d5 accuracy change <= -1 point", + "alternative": "mean d60-minus-d5 accuracy change > -1 point", + "predeclared_margin_points": margin, + "degrees_of_freedom": len(accuracy_changes) - 1, + "t_statistic": statistic, + "one_sided_p": one_sided_p, + "one_sided_95_lower_bound_points": one_sided_lower, + "two_sided_95_interval_points": [ + mean_change - two_sided_radius, + mean_change + two_sided_radius], + "reject_drop_at_least_1pt_alpha_0p05": one_sided_p < 0.05, + } if name != "BP": alignment_changes = [ early_alignment(deep[s]) - early_alignment(shallow[s]) for s in seeds @@ -810,6 +841,9 @@ depth and initialization seed. **c,** Mean per-sample cosine between the teachin descent direction, averaged over the earliest third of hidden layers. Exact gradients are used only for this diagnostic, never for local learning. “Scaling” here means preserving useful accuracy and credit assignment as depth grows; flattened-CIFAR accuracy itself does not increase with depth. +Against the predeclared `-1`-point retention margin, the paired one-sided depth-5-to-60 test rejects +a drop of at least one point for SDIL (`p=0.0037`) but not DFA (`p=0.2310`); results for every +method are retained in the figure manifest. **Figure 3 | Somato-dendritic innovation is necessary under predictable apical traffic.** Mean ± sample standard deviation across five seeds on MNIST, using depth-3, width-256 networks for 15 -- cgit v1.2.3