summaryrefslogtreecommitdiff
path: root/sdil
diff options
context:
space:
mode:
authorYurenHao0426 <Blackhao0426@gmail.com>2026-07-22 02:01:05 -0500
committerYurenHao0426 <Blackhao0426@gmail.com>2026-07-22 02:01:05 -0500
commit319b23afedb1b6db66b360f70350d4b8e7d72237 (patch)
tree2d9f9451ef34d0f563c70fc5298b73024919defa /sdil
parente60afe651eb1fbe60397413af4d0df43cf22f1d0 (diff)
docs: remove unsupported synthetic depth guarantees
Diffstat (limited to 'sdil')
-rw-r--r--sdil/data.py28
1 files changed, 13 insertions, 15 deletions
diff --git a/sdil/data.py b/sdil/data.py
index ad51f59..abaabe5 100644
--- a/sdil/data.py
+++ b/sdil/data.py
@@ -202,11 +202,10 @@ def make_teacher_student(n_in=128, n_classes=10, t_depth=8, t_width=64,
n_train=50000, n_test=10000, seed=0, residual=True,
t_scale=1.5, batch_size=128, device="cpu"):
"""Deep teacher-student classification. Labels = argmax of a fixed random
- deep (residual) teacher net. Students are WIDTH-LIMITED, so depth is the
- resource that matters: a shallow student cannot compose enough nonlinearity
- to match a deep teacher, a deep one can. This is the standard construction
- for a task where accuracy genuinely GROWS with model depth — unlike
- flattened-CIFAR MLPs, which are architecture-bottlenecked and depth-flat."""
+ deep (residual) teacher net. This is a candidate controlled composition
+ task, not a guarantee that the selected student width, optimizer, and hard
+ labels make added depth useful. Every configuration must pass the BP
+ useful-depth screen before it is used for a credit-assignment claim."""
from .core import SDILNet
tsizes = [n_in] + [t_width] * t_depth + [n_classes]
teacher = SDILNet(tsizes, act="tanh", device=device, seed=seed + 999,
@@ -223,13 +222,13 @@ def make_teacher_student(n_in=128, n_classes=10, t_depth=8, t_width=64,
def make_hierarchical(levels=6, n_classes=10, n_train=50000, n_test=10000,
seed=0, batch_size=128, device="cpu"):
- """Hierarchical compositional target with PROVABLE depth-dependence. Input
+ """Hierarchical compositional target. Input
has 2^levels features; C independent binary trees each fold adjacent pairs
with a fixed random nonlinear combiner v' = tanh(a·l + b·r + c·l·r) (the
- l·r product is the depth-hard bit). Label = argmax over the C tree roots.
- A depth-d net can only realise ~d tree levels, so accuracy GROWS with depth
- up to `levels` — the regime where scaling the model actually helps, and where
- a rule with poor deep credit assignment (DFA) cannot follow."""
+ l·r product is the composition-heavy term). Label = argmax over the C tree
+ roots. The generator is deep, but a dense shallow student can approximate
+ it with sufficient width; no depth lower bound is claimed for the student
+ family used here. BP must empirically pass a predeclared depth-gain gate."""
n_in = 2 ** levels
g = torch.Generator(device="cpu").manual_seed(seed + 31)
# per-class, per-level, per-node combiner params
@@ -273,11 +272,10 @@ def make_tentmap(levels=8, n_in=4, n_train=50000, n_test=10000, seed=0,
batch_size=128, device="cpu"):
"""Telgarsky depth-separation task. Label = [tent^levels(x0) > 0.5], where
tent(v)=1-|2v-1| self-composed `levels` times produces 2^levels oscillations.
- A depth-d ReLU net can realise ~d tent compositions, resolving only ~2^d of
- the 2^levels teeth, so TEST ACCURACY PROVABLY RISES WITH DEPTH up to `levels`.
- This is the regime where scaling the model genuinely buys accuracy, and where
- a rule with poor deep credit assignment (DFA/FA) cannot build the composition
- and stalls. Extra input dims are distractors (must be ignored)."""
+ Compositional ReLU constructions have a depth-efficiency separation on this
+ target, but ordinary random-initialized SGD is not guaranteed to discover
+ them. The experiment is therefore only eligible for a scaling claim after
+ BP demonstrates a held-out depth gain. Extra input dims are distractors."""
g = torch.Generator(device="cpu").manual_seed(seed + 7)
xtr = torch.rand(n_train, n_in, generator=g).to(device)
xte = torch.rand(n_test, n_in, generator=g).to(device)