From 319b23afedb1b6db66b360f70350d4b8e7d72237 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 02:01:05 -0500 Subject: docs: remove unsupported synthetic depth guarantees --- sdil/data.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) (limited to 'sdil') diff --git a/sdil/data.py b/sdil/data.py index ad51f59..abaabe5 100644 --- a/sdil/data.py +++ b/sdil/data.py @@ -202,11 +202,10 @@ def make_teacher_student(n_in=128, n_classes=10, t_depth=8, t_width=64, n_train=50000, n_test=10000, seed=0, residual=True, t_scale=1.5, batch_size=128, device="cpu"): """Deep teacher-student classification. Labels = argmax of a fixed random - deep (residual) teacher net. Students are WIDTH-LIMITED, so depth is the - resource that matters: a shallow student cannot compose enough nonlinearity - to match a deep teacher, a deep one can. This is the standard construction - for a task where accuracy genuinely GROWS with model depth — unlike - flattened-CIFAR MLPs, which are architecture-bottlenecked and depth-flat.""" + deep (residual) teacher net. This is a candidate controlled composition + task, not a guarantee that the selected student width, optimizer, and hard + labels make added depth useful. Every configuration must pass the BP + useful-depth screen before it is used for a credit-assignment claim.""" from .core import SDILNet tsizes = [n_in] + [t_width] * t_depth + [n_classes] teacher = SDILNet(tsizes, act="tanh", device=device, seed=seed + 999, @@ -223,13 +222,13 @@ def make_teacher_student(n_in=128, n_classes=10, t_depth=8, t_width=64, def make_hierarchical(levels=6, n_classes=10, n_train=50000, n_test=10000, seed=0, batch_size=128, device="cpu"): - """Hierarchical compositional target with PROVABLE depth-dependence. Input + """Hierarchical compositional target. Input has 2^levels features; C independent binary trees each fold adjacent pairs with a fixed random nonlinear combiner v' = tanh(a·l + b·r + c·l·r) (the - l·r product is the depth-hard bit). Label = argmax over the C tree roots. - A depth-d net can only realise ~d tree levels, so accuracy GROWS with depth - up to `levels` — the regime where scaling the model actually helps, and where - a rule with poor deep credit assignment (DFA) cannot follow.""" + l·r product is the composition-heavy term). Label = argmax over the C tree + roots. The generator is deep, but a dense shallow student can approximate + it with sufficient width; no depth lower bound is claimed for the student + family used here. BP must empirically pass a predeclared depth-gain gate.""" n_in = 2 ** levels g = torch.Generator(device="cpu").manual_seed(seed + 31) # per-class, per-level, per-node combiner params @@ -273,11 +272,10 @@ def make_tentmap(levels=8, n_in=4, n_train=50000, n_test=10000, seed=0, batch_size=128, device="cpu"): """Telgarsky depth-separation task. Label = [tent^levels(x0) > 0.5], where tent(v)=1-|2v-1| self-composed `levels` times produces 2^levels oscillations. - A depth-d ReLU net can realise ~d tent compositions, resolving only ~2^d of - the 2^levels teeth, so TEST ACCURACY PROVABLY RISES WITH DEPTH up to `levels`. - This is the regime where scaling the model genuinely buys accuracy, and where - a rule with poor deep credit assignment (DFA/FA) cannot build the composition - and stalls. Extra input dims are distractors (must be ignored).""" + Compositional ReLU constructions have a depth-efficiency separation on this + target, but ordinary random-initialized SGD is not guaranteed to discover + them. The experiment is therefore only eligible for a scaling claim after + BP demonstrates a held-out depth gain. Extra input dims are distractors.""" g = torch.Generator(device="cpu").manual_seed(seed + 7) xtr = torch.rand(n_train, n_in, generator=g).to(device) xte = torch.rand(n_test, n_in, generator=g).to(device) -- cgit v1.2.3