From 6db80927045e6c8ca91df9dd505f07463cc3d73d Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Mon, 20 Jul 2026 06:48:29 -0500 Subject: =?UTF-8?q?RESULT=2054:=20=E4=B8=8A=E7=95=8C=E2=88=9D1/=CF=83?= =?UTF-8?q?=C2=B2=E4=B8=8B=E6=B2=89(=CF=83242=E2=86=92473=E7=BF=BB?= =?UTF-8?q?=E5=80=8D,=CE=B2*=E4=BB=8E0.12=E8=B7=8C=E5=88=B0<0.03),?= =?UTF-8?q?=E5=9B=BA=E5=AE=9A=CE=B2=3D=E5=81=9C=E6=BB=9E=E9=9D=9E=E5=AE=89?= =?UTF-8?q?=E5=85=A8(plain2=E6=94=B6=E5=AE=9889%skip,best=E5=85=B6?= =?UTF-8?q?=E5=AE=9E=E6=98=AF200k=E6=88=90=E7=BB=A9);=20K=3D30=E5=AF=86?= =?UTF-8?q?=E6=8E=A2=E9=92=88=E6=8E=A8=E7=BF=BBCodex=E7=9A=84=CE=B2*=3D0.7?= =?UTF-8?q?(K=3D8=E8=AF=AF=E5=88=A4=E6=85=A2=E5=8F=91=E6=95=A3=E4=B8=BA?= =?UTF-8?q?=E6=94=B6=E6=95=9B);=20=CF=83-scaling=E5=8E=9F=E8=AE=BE?= =?UTF-8?q?=E8=AE=A1=E5=AF=B9=E4=BD=86=E8=A2=ABfloor=E9=92=89=E6=AD=BB+?= =?UTF-8?q?=E6=97=A9=E6=9C=9F=CF=83=E6=9A=B4=E6=B6=A8=E5=BC=A0=E5=8A=9B;?= =?UTF-8?q?=20--beta=5Fcos=5Fmin=E5=AE=9E=E8=A3=85(=E6=8C=89=E8=BF=9B?= =?UTF-8?q?=E7=A8=8Bcosine=E9=99=8D=CE=B2=E8=B7=9F=E8=B8=AA=E4=B8=8B?= =?UTF-8?q?=E6=B2=89=E4=B8=8A=E7=95=8C);=20betacos=E7=8E=8B=E5=86=A02e-2?= =?UTF-8?q?=E2=86=928e-4=E5=9C=A8=E9=A3=9E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014FAPDWQ49M5Ye3NpTndTpn --- ep_run/casc_eq_train.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'ep_run/casc_eq_train.py') diff --git a/ep_run/casc_eq_train.py b/ep_run/casc_eq_train.py index c77b92c..86a8e06 100644 --- a/ep_run/casc_eq_train.py +++ b/ep_run/casc_eq_train.py @@ -29,6 +29,10 @@ ap.add_argument('--compile', action='store_true') # torch.compile each blo ap.add_argument('--sig_every', type=int, default=25) # tok-sigma refresh interval (amortized) ap.add_argument('--beta_floor', type=float, default=0.0) # >0: floor beta_t (anti finite-beta SNR collapse at depth) ap.add_argument('--beta_fixed', action='store_true') # disable sig^2 schedule, hold beta_t = args.beta constant +ap.add_argument('--beta_cos_min', type=float, default=0.0) # >0: cosine-descend beta from --beta to this over + # --steps (tracks the sinking ceiling by progress; + # bypasses sigma-scaling AND floor). Set --beta to the + # early value (below early ceiling ~0.12). ap.add_argument('--cosine', action='store_true') # warmup then cosine decay to lr_min_ratio*lr over --steps (long runs) ap.add_argument('--lr_min_ratio', type=float, default=0.1) ap.add_argument('--qk_norm', action='store_true') # RMS-norm q,k per head before scores (OLMo2-style; bounds logits, analog-friendly) @@ -472,9 +476,18 @@ def ep_step(x, y): if SIG0 is None: SIG0 = args.sig0 if args.sig0 > 0 else sig beta_t = args.beta * GOV['bscale'] * (SIG0 * SIG0) / max(sig * sig, 1e-9) if args.beta_fixed: beta_t = args.beta * GOV['bscale'] - fl = args.beta_floor - if args.bf_late > 0.0 and GOV.get('step', 0) >= args.bf_late_at: fl = args.bf_late - if fl > 0.0: beta_t = max(beta_t, fl) + if args.beta_cos_min > 0: + # SCHEDULED beta descent (07-20): the ceiling sinks ~1/sigma^2 as training sharpens the + # model (sigma 242->473); no fixed beta stays under it (endgame skip-stall). Descend beta + # by progress like LR — early large (below the high early ceiling), late small (below the + # sunk endgame ceiling). Skips sigma-scaling AND floor entirely. Conservative = safe: CE + # is flat across the in-corridor band, so undershoot costs nothing, overshoot skips. + prog = min(GOV.get('step', 0) / max(args.steps, 1), 1.0) + beta_t = args.beta_cos_min + 0.5 * (args.beta - args.beta_cos_min) * (1 + math.cos(math.pi * prog)) + else: + fl = args.beta_floor + if args.bf_late > 0.0 and GOV.get('step', 0) >= args.bf_late_at: fl = args.bf_late + if fl > 0.0: beta_t = max(beta_t, fl) if args.beta_ride > 1.0: # ride-v2(a): floor jumps must not compose with a pre-charged cap — rescale cap so the # EFFECTIVE beta is continuous across any floor change (the 0.09-at-20k bug, RESULT 37) -- cgit v1.2.3