diff options
| -rw-r--r-- | RESULTS.md | 7 | ||||
| -rw-r--r-- | experiments/plot_main_figures.py | 34 | ||||
| -rw-r--r-- | results/figs/main_figure_captions.md | 3 | ||||
| -rw-r--r-- | results/figs/main_figure_manifest.json | 58 |
4 files changed, 101 insertions, 1 deletions
@@ -88,6 +88,13 @@ depth grows 12x and wall time grows 9.87x (geometric mean). Its paired gap to BP 3.69 and 4.35 points at every tested depth. Flattened CIFAR itself is depth-flat, so this establishes preservation under depth, not a claim that extra depth improves the task. +The `-1`-point retention margin was fixed in `experiments/finalize_claims.sh` before the missing +depth cells were generated. A paired one-sided t-test rejects the null that SDIL's mean d5-to-d60 +change is at most `-1` point (`t(4)=5.030`, `p=0.0037`; one-sided 95% lower bound `-0.547`). The +same complete analysis rejects that null for BP (`p=0.0359`) and FA (`p=0.0321`) but not DFA +(`p=0.2310`; lower bound `-1.211`). This is an equivalence-to-the-prespecified-margin statement, +not a post-hoc test that the depth change is exactly zero. + The mean-based nondominated frontier among local methods is DFA-d5 (`5.01 s`, `37.77%`), FA-d5 (`5.72 s`, `38.92%`), then SDIL-d5 (`9.84 s`, `41.46%`). BP is shown only as a nonlocal reference. Thus SDIL occupies the high-accuracy end of the audited local-learning frontier; the deeper SDIL diff --git a/experiments/plot_main_figures.py b/experiments/plot_main_figures.py index 69009ab..a614343 100644 --- a/experiments/plot_main_figures.py +++ b/experiments/plot_main_figures.py @@ -17,6 +17,7 @@ import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter +from scipy.stats import t as student_t COLORS = { @@ -410,6 +411,36 @@ def audited_statistics(root): statistics.mean(math.log(value) for value in wall_ratios)), "d60_over_d5_wall_ratios": wall_ratios, } + if len(accuracy_changes) > 1: + mean_change, sd_change = mean_sd(accuracy_changes) + standard_error = sd_change / math.sqrt(len(accuracy_changes)) + margin = -1.0 + if standard_error > 0: + statistic = (mean_change - margin) / standard_error + one_sided_p = float(student_t.sf( + statistic, df=len(accuracy_changes) - 1)) + one_sided_lower = mean_change - float(student_t.ppf( + 0.95, df=len(accuracy_changes) - 1)) * standard_error + two_sided_radius = float(student_t.ppf( + 0.975, df=len(accuracy_changes) - 1)) * standard_error + else: + statistic = math.inf if mean_change > margin else -math.inf + one_sided_p = 0.0 if mean_change > margin else 1.0 + one_sided_lower = mean_change + two_sided_radius = 0.0 + endpoint["paired_accuracy_retention_test"] = { + "null": "mean d60-minus-d5 accuracy change <= -1 point", + "alternative": "mean d60-minus-d5 accuracy change > -1 point", + "predeclared_margin_points": margin, + "degrees_of_freedom": len(accuracy_changes) - 1, + "t_statistic": statistic, + "one_sided_p": one_sided_p, + "one_sided_95_lower_bound_points": one_sided_lower, + "two_sided_95_interval_points": [ + mean_change - two_sided_radius, + mean_change + two_sided_radius], + "reject_drop_at_least_1pt_alpha_0p05": one_sided_p < 0.05, + } if name != "BP": alignment_changes = [ early_alignment(deep[s]) - early_alignment(shallow[s]) for s in seeds @@ -810,6 +841,9 @@ depth and initialization seed. **c,** Mean per-sample cosine between the teachin descent direction, averaged over the earliest third of hidden layers. Exact gradients are used only for this diagnostic, never for local learning. “Scaling” here means preserving useful accuracy and credit assignment as depth grows; flattened-CIFAR accuracy itself does not increase with depth. +Against the predeclared `-1`-point retention margin, the paired one-sided depth-5-to-60 test rejects +a drop of at least one point for SDIL (`p=0.0037`) but not DFA (`p=0.2310`); results for every +method are retained in the figure manifest. **Figure 3 | Somato-dendritic innovation is necessary under predictable apical traffic.** Mean ± sample standard deviation across five seeds on MNIST, using depth-3, width-256 networks for 15 diff --git a/results/figs/main_figure_captions.md b/results/figs/main_figure_captions.md index 03c16c8..6b8562e 100644 --- a/results/figs/main_figure_captions.md +++ b/results/figs/main_figure_captions.md @@ -22,6 +22,9 @@ depth and initialization seed. **c,** Mean per-sample cosine between the teachin descent direction, averaged over the earliest third of hidden layers. Exact gradients are used only for this diagnostic, never for local learning. “Scaling” here means preserving useful accuracy and credit assignment as depth grows; flattened-CIFAR accuracy itself does not increase with depth. +Against the predeclared `-1`-point retention margin, the paired one-sided depth-5-to-60 test rejects +a drop of at least one point for SDIL (`p=0.0037`) but not DFA (`p=0.2310`); results for every +method are retained in the figure manifest. **Figure 3 | Somato-dendritic innovation is necessary under predictable apical traffic.** Mean ± sample standard deviation across five seeds on MNIST, using depth-3, width-256 networks for 15 diff --git a/results/figs/main_figure_manifest.json b/results/figs/main_figure_manifest.json index 294cf94..46527d3 100644 --- a/results/figs/main_figure_manifest.json +++ b/results/figs/main_figure_manifest.json @@ -2373,7 +2373,21 @@ 8.59384509799933, 8.346117401221937, 8.787989553791133 - ] + ], + "paired_accuracy_retention_test": { + "null": "mean d60-minus-d5 accuracy change <= -1 point", + "alternative": "mean d60-minus-d5 accuracy change > -1 point", + "predeclared_margin_points": -1.0, + "degrees_of_freedom": 4, + "t_statistic": 2.4311488030619435, + "one_sided_p": 0.03594586798395181, + "one_sided_95_lower_bound_points": -0.8672859622437409, + "two_sided_95_interval_points": [ + -1.1531084453710267, + 1.3091084453710273 + ], + "reject_drop_at_least_1pt_alpha_0p05": true + } }, "FA": { "seeds": [ @@ -2393,6 +2407,20 @@ 6.7033463045694015, 9.101054953876158 ], + "paired_accuracy_retention_test": { + "null": "mean d60-minus-d5 accuracy change <= -1 point", + "alternative": "mean d60-minus-d5 accuracy change > -1 point", + "predeclared_margin_points": -1.0, + "degrees_of_freedom": 4, + "t_statistic": 2.5357066333490828, + "one_sided_p": 0.03213672421335003, + "one_sided_95_lower_bound_points": -0.9197283470326103, + "two_sided_95_interval_points": [ + -1.0478494586936888, + 0.05584945869368485 + ], + "reject_drop_at_least_1pt_alpha_0p05": true + }, "d60_minus_d5_early_alignment_mean": -0.052620660066604596, "d60_minus_d5_early_alignment_sd": 0.013629272999240246 }, @@ -2414,6 +2442,20 @@ 9.083443515320905, 8.984656533045191 ], + "paired_accuracy_retention_test": { + "null": "mean d60-minus-d5 accuracy change <= -1 point", + "alternative": "mean d60-minus-d5 accuracy change > -1 point", + "predeclared_margin_points": -1.0, + "degrees_of_freedom": 4, + "t_statistic": 0.8126587379130075, + "one_sided_p": 0.2310059494410654, + "one_sided_95_lower_bound_points": -1.211028859093041, + "two_sided_95_interval_points": [ + -1.3141444444474168, + -0.4258555555525847 + ], + "reject_drop_at_least_1pt_alpha_0p05": false + }, "d60_minus_d5_early_alignment_mean": -0.46736509441630913, "d60_minus_d5_early_alignment_sd": 0.03641498460995918 }, @@ -2435,6 +2477,20 @@ 9.78526636436608, 10.016996559591552 ], + "paired_accuracy_retention_test": { + "null": "mean d60-minus-d5 accuracy change <= -1 point", + "alternative": "mean d60-minus-d5 accuracy change > -1 point", + "predeclared_margin_points": -1.0, + "degrees_of_freedom": 4, + "t_statistic": 5.030200405607772, + "one_sided_p": 0.003666228038780643, + "one_sided_95_lower_bound_points": -0.5471142775513921, + "two_sided_95_interval_points": [ + -0.6478367613052975, + 0.21983676130529367 + ], + "reject_drop_at_least_1pt_alpha_0p05": true + }, "d60_minus_d5_early_alignment_mean": -0.46029999762773516, "d60_minus_d5_early_alignment_sd": 0.02065602739281337 } |