From b83947778e2c776f757a07d4719b7ce961d7ed55 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Fri, 3 Jul 2026 05:56:50 -0500 Subject: =?UTF-8?q?Initial=20commit:=20ept=20=E2=80=94=20backprop-free=20e?= =?UTF-8?q?quilibrium=20transformer=20(EP)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code (ep_run/), organized docs (docs/{method,campaign,hardware,outreach,paper}), analysis scripts (scripts/), ONBOARDING.md entry point. Large data/checkpoints git-ignored (share separately). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/eps_sweep_s3200.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 ep_run/eps_sweep_s3200.py (limited to 'ep_run/eps_sweep_s3200.py') diff --git a/ep_run/eps_sweep_s3200.py b/ep_run/eps_sweep_s3200.py new file mode 100644 index 0000000..3c26d73 --- /dev/null +++ b/ep_run/eps_sweep_s3200.py @@ -0,0 +1,29 @@ +import torch, pickle, math +from pathlib import Path +import lt_ep_train as L +from lt_ep_train import EQBlock +L.DD=Path('data/tinystories_bpe'); L.vocab=pickle.load(open(L.DD/'meta.pkl','rb'))['vocab_size'] +dev='cuda'; B=8; T=256; Ttot=400.0 # fixed "time" budget eps*N=Ttot so all eps cover same settling +torch.manual_seed(1234); idx,y=L.get_batch('val',B,T); idx=idx.to(dev) if hasattr(idx,'to') else idx +ck=torch.load('runs/redx_traj/s3200.pt',map_location=dev) +def relax_eps(eps): + N=min(int(Ttot/eps), 24000) + blk=EQBlock(512,16,256,256,s=1.0,c=1.0,attn_mode='thick'); blk.qknorm=True + with torch.no_grad(): + for p,w in zip(blk.allp,ck['allp']): p.copy_(w.to(dev)) + xin=blk.embed(idx).detach(); z=xin.clone(); ress=[] + for t in range(N): + z2=z+eps*blk.force(z,xin).detach() + r=(z2-z).norm().item()/(z.norm().item()+1e-9); ress.append(r); z=z2 + if not math.isfinite(r) or r>1e3: return ('DIVERGED',t,r,r) + tail=ress[-min(800,N//4):] + return (N, ress[-1], min(tail), max(tail)) +print("=== eps-sweep on redx s3200 (FULL attention, the cycling operator): Euler-artifact vs continuous instability ===") +print("eps*N=400 held fixed (same settling time). Cycle dies as eps shrinks => DISCRETE-EULER ARTIFACT (continuous ODE / analog HW is fine).") +for eps in [0.1, 0.05, 0.03, 0.02, 0.01]: + r=relax_eps(eps) + if r[0]=='DIVERGED': print(f" eps={eps}: DIVERGED at t={r[1]} r={r[2]:.2e}") + else: + N,last,tmin,tmax=r; osc=tmax-tmin + print(f" eps={eps}: N={N:5d} res(last)={last:.3e} tail[min={tmin:.2e},max={tmax:.2e}] osc={osc:.2e} {'CYCLE' if (osc>5e-4 and last>2e-3) else 'CONVERGED' if last<2e-3 else 'floored'}") +print("=== DONE ===") -- cgit v1.2.3