diff options
Diffstat (limited to 'ep_run/casc_eq_train.py')
| -rw-r--r-- | ep_run/casc_eq_train.py | 9 |
1 files changed, 7 insertions, 2 deletions
diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index d7cfdf7..0e357c8 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -49,6 +49,9 @@ ap.add_argument('--bsign_rand', action='store_true') # random-sign beta per ste ap.add_argument('--bf16', action='store_true') # cast model to bf16 (E-accumulation + tok_sigma stay fp32) — the x0.5 cost lever, GATE before production ap.add_argument('--amp', action='store_true') # PROPER mixed precision: autocast(bf16) matmuls, fp32 params/states/d/E — amp_gate.py PASSED 2026-07-12 (cos 0.9682 vs fp32 0.9687); --bf16 naive-cast stays DEAD (state quantization, RESULT 11) ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) +ap.add_argument('--dgain_rand', type=float, default=0.0) # >1: per-STEP log-uniform dgain_top in + # [1, this] (spread-spectrum probing of the + # decade-spread threshold distribution) ap.add_argument('--dgain_top', type=float, default=1.0) # amplify d in STATE FORMATION for blocks # >= L/2 (read cotangents stay true-d: 1st- # order exact; unlocks 2nd-order response of @@ -419,7 +422,7 @@ def relax(z0, zs, ins, outs, y, beta, K, x, bmask=None): else: i = prev.detach().requires_grad_(True) o = blocks[l](i, mask) - _dg = args.dgain_all * (args.dgain_top if l >= args.L // 2 else 1.0) + _dg = args.dgain_all * ((GOV.get('dgcur') or args.dgain_top) if l >= args.L // 2 else 1.0) znew = o.detach().float() + (_dg * d[l] if _dg != 1.0 else d[l]) # damped (under-relaxed) mixing: geta<1 restores contraction on stiff operators # (wall-2 toolkit); fixed point unchanged (z = z + geta*(o+d-z) <=> z = o+d) @@ -521,6 +524,8 @@ def ep_step(x, y): GOV['prev_floor'] = fl beta_t = beta_t * GOV.get('cap', 1.0) # wall-2 loop-gain cap OVERRIDES the floor (the ceiling # can sit below the floor near the wall; survival first) + if args.dgain_rand > 1.0: + GOV['dgcur'] = float(torch.exp(torch.rand((), generator=BGEN) * math.log(args.dgain_rand))) if args.beta_simple >= 1.0: # FULL ownership: no sigma-scaling, no floor/bf_late, no cap — beta = start * measured multiplier beta_t = args.beta * GOV.get('bsimp', 1.0) @@ -925,7 +930,7 @@ for step in range(start_step, args.steps + 1): with torch.no_grad(): GOV['w_rms'] = float(sum(p.float().pow(2).mean().sqrt() for p in all_params) / len(all_params)) _aux = {'gn': GOV.get('gn'), 'drift': GOV.get('drift'), 'sig': GOV.get('sig'), - 'res': GOV.get('res'), 'rho': GOV.get('rho'), + 'res': GOV.get('res'), 'rho': GOV.get('rho'), 'dgcur': GOV.get('dgcur'), 'clip_norm': GOV.get('clip_norm'), 'clip_fired': (None if GOV.get('clip_norm') is None else float(GOV['clip_norm'] > 1.0))} for k in ('act_rms', 'w_rms', 'mag_ep', 'mag_bp'): if GOV.get(k) is not None: _aux[k] = GOV[k] |