From d075af6e8ddcb067444f1c8b64cb294a75dd4e6a Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Thu, 30 Jul 2026 11:22:31 -0500 Subject: =?UTF-8?q?dgain=E5=87=A0=E4=BD=95profile=20+=20M1=E6=B3=84?= =?UTF-8?q?=E6=BC=8F=E8=B0=B1=E4=BB=AA(=E7=94=A8=E6=88=B7=E8=AE=BE?= =?UTF-8?q?=E8=AE=A1,07-30)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --dgain_geo r: 每层读出位移增益=r^l(首层恒1,不含总深度L,加深自动延伸); --dgain_geo_cap封顶。 用户提案"首层不变逐层x2"的实装,三臂电池(2^l | 2^l cap128 | 1.414^l)已排队于仪器ckpt。 --probe_dgspec N: 逐块顽固度谱仪。L_l(g)=mean_b[gEP_l(g)-gBP_l]同批配对差分(batch采样噪声 精确对消,残余=一致性偏差+估计器噪声/sqrtN); 主指标=在BP方向的带符号投影(范数在1e7维被噪声 膨胀,冒烟实测g8读700%,投影无此病且恰为致害分量); 附(-beta,g=1)臂分离奇偶(beta奇函数FD偏差 翻号,位移阈值泄漏为|位移|偶函数不翻); 同时记录每块自由相位移比RMS(d_l)/RMS(z_l)=无量纲律 候选横轴。fp32仪器级(bf16地板致盲,b15教训)。GOV['dgprofile']钩子仅探针置位,训练路径不触。 排队: dgspec C768(泄漏态s250000)+C512对照(应读~0) on GPU1, ladder后补扫C320/C192/C128 =5宽度x12块60点做g*_l x 位移比塌缩检验。 Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_eq_train.py | 88 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 2 deletions(-) (limited to 'ep_run/casc_eq_train.py') diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index 0e357c8..a9218a0 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -49,7 +49,12 @@ ap.add_argument('--bsign_rand', action='store_true') # random-sign beta per ste ap.add_argument('--bf16', action='store_true') # cast model to bf16 (E-accumulation + tok_sigma stay fp32) — the x0.5 cost lever, GATE before production ap.add_argument('--amp', action='store_true') # PROPER mixed precision: autocast(bf16) matmuls, fp32 params/states/d/E — amp_gate.py PASSED 2026-07-12 (cos 0.9682 vs fp32 0.9687); --bf16 naive-cast stays DEAD (state quantization, RESULT 11) ap.add_argument('--dtop_every', type=int, default=1) # 1 = exact (DEFAULT, BP-parity); 2 = fast mode (~20% cheaper, ~4% CE tax at high lr) -ap.add_argument('--dgain_rand', type=float, default=0.0) # >1: per-STEP log-uniform dgain_top in +ap.add_argument('--dgain_geo', type=float, default=0.0) # >0: per-layer geometric read-displacement + # gain = geo^l (layer 0 = x1), optionally capped +ap.add_argument('--dgain_geo_cap', type=float, default=0.0) # >0: cap for the geometric profile +ap.add_argument('--dgain_rand', type=float, default=0.0) +ap.add_argument('--probe_dgspec', type=int, default=0) # >0: M1 spectroscopy, value = n batches; exits before training +ap.add_argument('--probe_gains', default='1,2,4,8,16,32,64,128,256') # >1: per-STEP log-uniform dgain_top in # [1, this] (spread-spectrum probing of the # decade-spread threshold distribution) ap.add_argument('--dgain_top', type=float, default=1.0) # amplify d in STATE FORMATION for blocks @@ -422,7 +427,14 @@ def relax(z0, zs, ins, outs, y, beta, K, x, bmask=None): else: i = prev.detach().requires_grad_(True) o = blocks[l](i, mask) - _dg = args.dgain_all * ((GOV.get('dgcur') or args.dgain_top) if l >= args.L // 2 else 1.0) + _dgp = GOV.get('dgprofile') # probe-only per-layer profile; never set in training + if _dgp is not None: + _dg = _dgp.get(l, 1.0) + elif args.dgain_geo > 0: + _dg = args.dgain_geo ** l + if args.dgain_geo_cap > 0: _dg = min(_dg, args.dgain_geo_cap) + else: + _dg = args.dgain_all * ((GOV.get('dgcur') or args.dgain_top) if l >= args.L // 2 else 1.0) znew = o.detach().float() + (_dg * d[l] if _dg != 1.0 else d[l]) # damped (under-relaxed) mixing: geta<1 restores contraction on stiff operators # (wall-2 toolkit); fixed point unchanged (z = z + geta*(o+d-z) <=> z = o+d) @@ -835,6 +847,78 @@ if args.ddp_grad_test: import sys sys.exit(0) +if args.probe_dgspec > 0: + # M1 DGAIN SPECTROSCOPY: per-block leak vector vs uniform read-displacement gain. + # Paired design: L_l(g) = mean_b[gEP_l(g) - gBP_l] on the SAME batch — batch-sampling + # noise cancels exactly in the difference; what survives is the consistent bias (= the + # leak) plus estimator-internal noise shrinking 1/sqrt(NB). Extra (-beta, g=1) arm + # splits odd-in-beta FD bias from even-in-|displacement| leak. Run WITHOUT --amp: the + # bf16 floor is width-blind (b15 lesson); fp32 is the instrument-grade path. + assert WORLD == 1, 'probe_dgspec is single-GPU' + import json, sys + gains = [float(t) for t in args.probe_gains.split(',')] + cfgs = [(f'g{g:g}', g, +1.0) for g in gains] + [('g1neg', 1.0, -1.0)] + NB = args.probe_dgspec + bix = [] + for blk in blocks: + ids = {id(q) for q in blk.parameters()} + bix.append([i for i, p in enumerate(all_params) if id(p) in ids]) + accD = {k: [torch.zeros(sum(all_params[i].numel() for i in ix), device=dev) for ix in bix] + for k, _, _ in cfgs} + accO = {k: [torch.zeros_like(t) for t in accD[k]] for k, _, _ in cfgs} # odd-batch half (jackknife) + accB = [torch.zeros_like(t) for t in accD[cfgs[0][0]]] + disp = [0.0] * args.L # RMS(d_l)/RMS(z_l) at g=1,+beta + def _flat(gl, ix): + return torch.cat([(gl[i] if gl[i] is not None else torch.zeros_like(all_params[i])) + .reshape(-1).float() for i in ix]) + for bi_ in range(NB): + x, y = get_batch('train') + gbp = bp_gate(x, y) + bpfl = [_flat(gbp, ix) for ix in bix] + for j, t in enumerate(bpfl): accB[j] += t + for key, gA, sgn in cfgs: + GOV['dgprofile'] = {l: gA for l in range(args.L)} + bt = sgn * args.beta + z0, zs, ins, outs = free_states_graphed(x) + zp, lo = relax(z0, zs, ins, outs, y, bt, args.K, x) + if key == 'g1' : + with torch.no_grad(): + for l in range(args.L): + disp[l] += float(GOV['_last_d'][l].norm() / max(float(zp[l].norm()), 1e-12)) / NB + E = 0.0 + for z, o in zip(zp, lo): E = E + 0.5 * ((z.detach().float() - o.float()) ** 2).sum() + obj = E / (NBT * bt) + obj_loss(readout(zp[-1].detach()).reshape(-1, vocab), y.reshape(-1)) + gs = torch.autograd.grad(obj, all_params, allow_unused=True) + for j, ix in enumerate(bix): + d_ = _flat(gs, ix) - bpfl[j] + accD[key][j] += d_ + if bi_ % 2 == 1: accO[key][j] += d_ + GOV['dgprofile'] = None + if (bi_ + 1) % 16 == 0: print(f'[dgspec] batch {bi_+1}/{NB}', flush=True) + out = {'disp_ratio': disp} + for key, gA, sgn in cfgs: + rows = [] + for j in range(args.L): + mB, mD = accB[j] / NB, accD[key][j] / NB + ne, no_ = NB - NB // 2, NB // 2 + mDe, mDo = (accD[key][j] - accO[key][j]) / ne, accO[key][j] / max(no_, 1) + rows.append({'block': j, + 'rel_leak': float(mD.norm() / max(float(mB.norm()), 1e-12)), + 'proj_on_bp': float((mD @ mB) / max(float(mB.norm()) ** 2, 1e-24)), + 'half_split': float((mDe - mDo).norm() / max(float(mD.norm()), 1e-12))}) + out[key] = rows + # proj_on_bp is PRIMARY: the norm metric carries a positive noise-dimension bias + # (‖mean‖ >= ‖bias‖ inflated by residual noise over ~1e7 coords), the signed projection + # is unbiased and is exactly the damaging component; expect proj<0 (missing response). + print(f'[dgspec] {key:>7}: proj ' + ' '.join(f'{r["proj_on_bp"]:+.3f}' for r in rows) + + ' | rel ' + ' '.join(f'{r["rel_leak"]:.3f}' for r in rows), flush=True) + print('[dgspec] disp_ratio ' + ' '.join(f'{v:.2e}' for v in disp), flush=True) + Path('runs').mkdir(exist_ok=True) + with open(f'runs/dgspec_{args.tag}.json', 'w') as f: + json.dump({'args': {k: str(v) for k, v in vars(args).items()}, 'NB': NB, 'curves': out}, f) + print(f'[dgspec] DONE -> runs/dgspec_{args.tag}.json', flush=True) + sys.exit(0) + best, t0 = 1e9, time.time() skips = 0 for _ in range(start_step): sched.step() # advance LR schedule to the resumed step -- cgit v1.2.3