diff options
| -rw-r--r-- | docs/campaign/CASCADE_ABLATION_PLAN.md | 20 | ||||
| -rw-r--r-- | ep_run/casc_bp_train.py | 9 | ||||
| -rw-r--r-- | ep_run/casc_eq_train.py | 17 |
3 files changed, 43 insertions, 3 deletions
diff --git a/docs/campaign/CASCADE_ABLATION_PLAN.md b/docs/campaign/CASCADE_ABLATION_PLAN.md index 52c266e..0df3e72 100644 --- a/docs/campaign/CASCADE_ABLATION_PLAN.md +++ b/docs/campaign/CASCADE_ABLATION_PLAN.md @@ -568,6 +568,26 @@ direction), not training-under-fault; wave-2 = co-training with faults injected +### RESULT 69 (2026-07-29): SATURATION-BLINDNESS REFUTED BY ITS OWN TREATMENT TEST; the +leak survives exclusion #11; dgain (displacement-only amplification) is the live arm. +Battery 4 (beta dose): leak INVERSE in beta — 1.5e-3: 0.0615, 3e-3: 0.0462, 6e-3: 0.0352 +(~1/beta additive signature). This motivated the saturation-blindness hypothesis (sharp +softmax switches invisible to finite displacement; the sibling Hopfield-EP "saturated units +EP!=BPTT 80-130deg" lesson at transformer scale). +Battery 5 (user-approved, matched-cap BP bar): logit softcap 30/50 (Gemma-2 style; firing +certificate: cap=1 destroys the model, val 6.45) closes 0.0000 of the leak — relative leak +0.0462 IDENTICAL to 4 decimals. Saturation-at-the-logit level is NOT the mechanism. BONUS: +the cap's own price under BP ~ 0 in-window (allbp+cap30 3.5205 vs allbp 3.5206) — the user's +"cap hurts performance" concern priced at ~zero here (Gemma prior confirmed). +SURVIVING CONSTRAINTS: top-half (100%), attn-dominant (2/3), ~1/beta, immune to: logit cap, +odd/even bias, read noise, K, precision, head surgery, bottom blocks, magnitude/clip, Muon, +direction (FA argument). Threshold nonlinearity still indicated by 1/beta but NOT at softmax +logits — candidates: ffn gates (mixffn arm measuring its share), norms, qk-norm. +Battery 6 IN FLIGHT: leak_dg4top (top-half state-formation d x4, cotangents true-d; smoke +shows the price: gate cos 0.83 from anchor second-order — CE judges), leak_dg2all +(displacement-vs-force discriminator vs the b6d arm which doubled BOTH: b6d 3.5558), +leak_mixffn (completes the R44-style partition additivity). + ### RESULT 68 (2026-07-28): LEAK LOCALIZED — 100% in the TOP-HALF block gradients, ~2/3 attention; the SAME map as R44 at 72M, 6x stronger at width. One structure, two scales. Battery 3 (guaranteed-signal partition, bars allbp 3.5206 / EP 3.5668): mixtop closes 100% diff --git a/ep_run/casc_bp_train.py b/ep_run/casc_bp_train.py index 42acf16..4ecfee9 100644 --- a/ep_run/casc_bp_train.py +++ b/ep_run/casc_bp_train.py @@ -13,6 +13,7 @@ ap.add_argument('--B', type=int, default=24); ap.add_argument('--steps', type=in ap.add_argument('--lr', type=float, default=3e-4); ap.add_argument('--warmup', type=int, default=200) ap.add_argument('--seed', type=int, default=0) ap.add_argument('--save_every', type=int, default=500); ap.add_argument('--log', type=int, default=200) +ap.add_argument('--logit_cap', type=float, default=0.0) # >0: Gemma-2-style attn logit softcap ap.add_argument('--watch_every', type=int, default=2000) # wandb-only telemetry: weight/act RMS ap.add_argument('--wandb', default='auto') # ON BY DEFAULT; 'auto' = per-regime project (ept-fineweb-72m / ept-tinystories-42m); --wandb '' to disable ap.add_argument('--wandb_run', default='') @@ -114,7 +115,13 @@ class Olmo2Attn(nn.Module): q = self.rope(q.view(B, T, self.H, self.hd).transpose(1, 2)) k = self.rope(k.view(B, T, self.H, self.hd).transpose(1, 2)) v = v.view(B, T, self.H, self.hd).transpose(1, 2) - y = F.scaled_dot_product_attention(q, k, v, is_causal=True) + if args.logit_cap > 0: + lg = (q @ k.transpose(-2, -1)) * (self.hd ** -0.5) + lg = args.logit_cap * torch.tanh(lg / args.logit_cap) + cm = torch.ones(T, T, dtype=torch.bool, device=x.device).tril() + y = lg.masked_fill(~cm, float('-inf')).softmax(-1) @ v + else: + y = F.scaled_dot_product_attention(q, k, v, is_causal=True) return self.proj(y.transpose(1, 2).contiguous().view(B, T, C)) class Olmo2Block(nn.Module): diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 07e2ad4..d7cfdf7 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -49,6 +49,12 @@ ap.add_argument('--bsign_rand', action='store_true') # random-sign beta per ste ap.add_argument('--bf16', action='store_true') # cast model to bf16 (E-accumulation + tok_sigma stay fp32) — the x0.5 cost lever, GATE before production ap.add_argument('--amp', action='store_true') # PROPER mixed precision: autocast(bf16) matmuls, fp32 params/states/d/E — amp_gate.py PASSED 2026-07-12 (cos 0.9682 vs fp32 0.9687); --bf16 naive-cast stays DEAD (state quantization, RESULT 11) ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) +ap.add_argument('--dgain_top', type=float, default=1.0) # amplify d in STATE FORMATION for blocks + # >= L/2 (read cotangents stay true-d: 1st- + # order exact; unlocks 2nd-order response of + # threshold nonlinearities without global beta) +ap.add_argument('--dgain_all', type=float, default=1.0) # same, all blocks (displacement-vs-force probe) +ap.add_argument('--logit_cap', type=float, default=0.0) # >0: Gemma-2-style attn logit softcap ap.add_argument('--head_lr_mult', type=float, default=1.0) # W_out Adam-group LR multiplier (C768 # head-throttle compensation, RESULT 66) ap.add_argument('--res_gate', type=float, default=0.02) # legality residual threshold; 0.02 was calibrated @@ -258,7 +264,13 @@ class Olmo2Attn(nn.Module): q = self.rope(q.view(B, T, self.H, self.hd).transpose(1, 2)) k = self.rope(k.view(B, T, self.H, self.hd).transpose(1, 2)) v = v.view(B, T, self.H, self.hd).transpose(1, 2) - y = F.scaled_dot_product_attention(q, k, v, is_causal=True) + if args.logit_cap > 0: + lg = (q @ k.transpose(-2, -1)) * (self.hd ** -0.5) + lg = args.logit_cap * torch.tanh(lg / args.logit_cap) + cm = torch.ones(T, T, dtype=torch.bool, device=x.device).tril() + y = lg.masked_fill(~cm, float('-inf')).softmax(-1) @ v + else: + y = F.scaled_dot_product_attention(q, k, v, is_causal=True) return self.proj(y.transpose(1, 2).contiguous().view(B, T, C)) class Olmo2Block(nn.Module): @@ -407,7 +419,8 @@ def relax(z0, zs, ins, outs, y, beta, K, x, bmask=None): else: i = prev.detach().requires_grad_(True) o = blocks[l](i, mask) - znew = o.detach().float() + d[l] + _dg = args.dgain_all * (args.dgain_top if l >= args.L // 2 else 1.0) + znew = o.detach().float() + (_dg * d[l] if _dg != 1.0 else d[l]) # damped (under-relaxed) mixing: geta<1 restores contraction on stiff operators # (wall-2 toolkit); fixed point unchanged (z = z + geta*(o+d-z) <=> z = o+d) mixed = znew if g_eff >= 1.0 else (zs[l] + g_eff * (znew - zs[l])) |