diff options
| -rw-r--r-- | experiments/analyze_depth_tasks.py | 32 | ||||
| -rw-r--r-- | sdil/data.py | 28 |
2 files changed, 34 insertions, 26 deletions
diff --git a/experiments/analyze_depth_tasks.py b/experiments/analyze_depth_tasks.py index f1a8e19..b0df690 100644 --- a/experiments/analyze_depth_tasks.py +++ b/experiments/analyze_depth_tasks.py @@ -24,27 +24,37 @@ def main(): key = (args["dataset"], args["mode"], args["width"], args["depth"]) groups.setdefault(key, []).append(row) - print("| task | method | width | depth | task/model runs | validation acc (%) |") - print("|:---|:---|---:|---:|---:|---:|") + print("| task | method | width | depth | task/model runs | last val (%) | best val (%) |") + print("|:---|:---|---:|---:|---:|---:|---:|") by_curve = {} for key in sorted(groups): rows = groups[key] - values = [100 * row["final"]["val_acc"] for row in rows] - mean = statistics.mean(values) - sd = statistics.stdev(values) if len(values) > 1 else None - value = f"{mean:.3f}" if sd is None else f"{mean:.3f} ± {sd:.3f}" - print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | {value} |") - by_curve.setdefault(key[:3], {})[key[3]] = mean + last_values = [100 * row["final"]["val_acc"] for row in rows] + best_values = [100 * max(step["val_acc"] for step in row["steps"] + if "val_acc" in step) for row in rows] + last_mean = statistics.mean(last_values) + best_mean = statistics.mean(best_values) + last_sd = statistics.stdev(last_values) if len(last_values) > 1 else None + best_sd = statistics.stdev(best_values) if len(best_values) > 1 else None + last_text = (f"{last_mean:.3f}" if last_sd is None + else f"{last_mean:.3f} ± {last_sd:.3f}") + best_text = (f"{best_mean:.3f}" if best_sd is None + else f"{best_mean:.3f} ± {best_sd:.3f}") + print(f"| {key[0]} | {key[1].upper()} | {key[2]} | {key[3]} | {len(rows)} | " + f"{last_text} | {best_text} |") + by_curve.setdefault(key[:3], {})[key[3]] = (last_mean, best_mean) print() for key, curve in sorted(by_curve.items()): if len(curve) < 2: continue shallow, deep = min(curve), max(curve) - gain = curve[deep] - curve[shallow] - verdict = "PASS" if gain >= 5.0 else "FAIL" + last_gain = curve[deep][0] - curve[shallow][0] + best_gain = curve[deep][1] - curve[shallow][1] + verdict = "PASS" if best_gain >= 5.0 else "FAIL" print(f"{key[0]} {key[1].upper()} w{key[2]}: d{shallow}->d{deep} " - f"gain {gain:+.3f} points [{verdict} C2 BP-screen threshold]") + f"last gain {last_gain:+.3f}, best-epoch gain {best_gain:+.3f} points " + f"[{verdict} C2 BP-screen threshold]") if __name__ == "__main__": diff --git a/sdil/data.py b/sdil/data.py index ad51f59..abaabe5 100644 --- a/sdil/data.py +++ b/sdil/data.py @@ -202,11 +202,10 @@ def make_teacher_student(n_in=128, n_classes=10, t_depth=8, t_width=64, n_train=50000, n_test=10000, seed=0, residual=True, t_scale=1.5, batch_size=128, device="cpu"): """Deep teacher-student classification. Labels = argmax of a fixed random - deep (residual) teacher net. Students are WIDTH-LIMITED, so depth is the - resource that matters: a shallow student cannot compose enough nonlinearity - to match a deep teacher, a deep one can. This is the standard construction - for a task where accuracy genuinely GROWS with model depth — unlike - flattened-CIFAR MLPs, which are architecture-bottlenecked and depth-flat.""" + deep (residual) teacher net. This is a candidate controlled composition + task, not a guarantee that the selected student width, optimizer, and hard + labels make added depth useful. Every configuration must pass the BP + useful-depth screen before it is used for a credit-assignment claim.""" from .core import SDILNet tsizes = [n_in] + [t_width] * t_depth + [n_classes] teacher = SDILNet(tsizes, act="tanh", device=device, seed=seed + 999, @@ -223,13 +222,13 @@ def make_teacher_student(n_in=128, n_classes=10, t_depth=8, t_width=64, def make_hierarchical(levels=6, n_classes=10, n_train=50000, n_test=10000, seed=0, batch_size=128, device="cpu"): - """Hierarchical compositional target with PROVABLE depth-dependence. Input + """Hierarchical compositional target. Input has 2^levels features; C independent binary trees each fold adjacent pairs with a fixed random nonlinear combiner v' = tanh(a·l + b·r + c·l·r) (the - l·r product is the depth-hard bit). Label = argmax over the C tree roots. - A depth-d net can only realise ~d tree levels, so accuracy GROWS with depth - up to `levels` — the regime where scaling the model actually helps, and where - a rule with poor deep credit assignment (DFA) cannot follow.""" + l·r product is the composition-heavy term). Label = argmax over the C tree + roots. The generator is deep, but a dense shallow student can approximate + it with sufficient width; no depth lower bound is claimed for the student + family used here. BP must empirically pass a predeclared depth-gain gate.""" n_in = 2 ** levels g = torch.Generator(device="cpu").manual_seed(seed + 31) # per-class, per-level, per-node combiner params @@ -273,11 +272,10 @@ def make_tentmap(levels=8, n_in=4, n_train=50000, n_test=10000, seed=0, batch_size=128, device="cpu"): """Telgarsky depth-separation task. Label = [tent^levels(x0) > 0.5], where tent(v)=1-|2v-1| self-composed `levels` times produces 2^levels oscillations. - A depth-d ReLU net can realise ~d tent compositions, resolving only ~2^d of - the 2^levels teeth, so TEST ACCURACY PROVABLY RISES WITH DEPTH up to `levels`. - This is the regime where scaling the model genuinely buys accuracy, and where - a rule with poor deep credit assignment (DFA/FA) cannot build the composition - and stalls. Extra input dims are distractors (must be ignored).""" + Compositional ReLU constructions have a depth-efficiency separation on this + target, but ordinary random-initialized SGD is not guaranteed to discover + them. The experiment is therefore only eligible for a scaling claim after + BP demonstrates a held-out depth gain. Extra input dims are distractors.""" g = torch.Generator(device="cpu").manual_seed(seed + 7) xtr = torch.rand(n_train, n_in, generator=g).to(device) xte = torch.rand(n_test, n_in, generator=g).to(device) |