diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/plot_main_figures.py | 34 |
1 files changed, 34 insertions, 0 deletions
diff --git a/experiments/plot_main_figures.py b/experiments/plot_main_figures.py index 69009ab..a614343 100644 --- a/experiments/plot_main_figures.py +++ b/experiments/plot_main_figures.py @@ -17,6 +17,7 @@ import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.ticker import FuncFormatter +from scipy.stats import t as student_t COLORS = { @@ -410,6 +411,36 @@ def audited_statistics(root): statistics.mean(math.log(value) for value in wall_ratios)), "d60_over_d5_wall_ratios": wall_ratios, } + if len(accuracy_changes) > 1: + mean_change, sd_change = mean_sd(accuracy_changes) + standard_error = sd_change / math.sqrt(len(accuracy_changes)) + margin = -1.0 + if standard_error > 0: + statistic = (mean_change - margin) / standard_error + one_sided_p = float(student_t.sf( + statistic, df=len(accuracy_changes) - 1)) + one_sided_lower = mean_change - float(student_t.ppf( + 0.95, df=len(accuracy_changes) - 1)) * standard_error + two_sided_radius = float(student_t.ppf( + 0.975, df=len(accuracy_changes) - 1)) * standard_error + else: + statistic = math.inf if mean_change > margin else -math.inf + one_sided_p = 0.0 if mean_change > margin else 1.0 + one_sided_lower = mean_change + two_sided_radius = 0.0 + endpoint["paired_accuracy_retention_test"] = { + "null": "mean d60-minus-d5 accuracy change <= -1 point", + "alternative": "mean d60-minus-d5 accuracy change > -1 point", + "predeclared_margin_points": margin, + "degrees_of_freedom": len(accuracy_changes) - 1, + "t_statistic": statistic, + "one_sided_p": one_sided_p, + "one_sided_95_lower_bound_points": one_sided_lower, + "two_sided_95_interval_points": [ + mean_change - two_sided_radius, + mean_change + two_sided_radius], + "reject_drop_at_least_1pt_alpha_0p05": one_sided_p < 0.05, + } if name != "BP": alignment_changes = [ early_alignment(deep[s]) - early_alignment(shallow[s]) for s in seeds @@ -810,6 +841,9 @@ depth and initialization seed. **c,** Mean per-sample cosine between the teachin descent direction, averaged over the earliest third of hidden layers. Exact gradients are used only for this diagnostic, never for local learning. “Scaling” here means preserving useful accuracy and credit assignment as depth grows; flattened-CIFAR accuracy itself does not increase with depth. +Against the predeclared `-1`-point retention margin, the paired one-sided depth-5-to-60 test rejects +a drop of at least one point for SDIL (`p=0.0037`) but not DFA (`p=0.2310`); results for every +method are retained in the figure manifest. **Figure 3 | Somato-dendritic innovation is necessary under predictable apical traffic.** Mean ± sample standard deviation across five seeds on MNIST, using depth-3, width-256 networks for 15 |