From 25ee60c8277ba82b9cb6471471b1a727e0712ea7 Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 8 Apr 2026 03:22:36 -0500 Subject: =?UTF-8?q?=CE=BB=20sweep=20on=20penalty=20strength:=20lam=20?= =?UTF-8?q?=E2=88=88=20{1e-4,=201e-2,=201e-1}=20cos=20+=20rho=20results?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 19's #5 recommendation. Major new finding for the paper: | lam | acc | ||h_L|| | ||g_2|| | deep cos | deep rho | |-------|------:|--------:|--------:|---------:|---------:| | 0 | 0.308 | 4e8 | 5e-10 | -0.008 | -0.003 | | 1e-4 | 0.359 | 2.4e4 | 6.3e-7 | -0.022 | -0.004 | | 1e-2 | 0.363 | 4e4 | 1e-6 | +0.155 | +0.080 | | 1e-1 | 0.349 | 1.2e4 | 1.6e-6 | +0.131 | +0.067 | KEY: at lam=1e-4 the residual stream is contained AND ||g|| is healthy (mode 1 ALLEVIATED), but deep cos and rho are still essentially zero (mode 2 NOT alleviated). This is independent dissociation of the two modes via penalty strength: at weak penalty you get mode 1 fix WITHOUT mode 2 fix. Both metrics (cos, rho) agree at every lambda. Penalty strength has a non-monotonic effect on mode 2 alleviation: - lam=1e-4: too weak, mode 2 not alleviated (cos ~0) - lam=1e-2: sweet spot, cos +0.16, rho +0.08 - lam=1e-1: slightly over-constrained, cos +0.13, rho +0.07 This is the 7th independent validation of the two-mode separation, and the strongest one because it shows mode 1 alleviation WITHOUT mode 2 alleviation — the modes do not even respond to the same intervention strength. --- results/dfa_pen_short/dfa_pen_lam0.0001_s42.json | 43 ++++++++++++++++++++++++ results/dfa_pen_short/dfa_pen_lam0.1_s42.json | 43 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 results/dfa_pen_short/dfa_pen_lam0.0001_s42.json create mode 100644 results/dfa_pen_short/dfa_pen_lam0.1_s42.json (limited to 'results/dfa_pen_short') diff --git a/results/dfa_pen_short/dfa_pen_lam0.0001_s42.json b/results/dfa_pen_short/dfa_pen_lam0.0001_s42.json new file mode 100644 index 0000000..da25733 --- /dev/null +++ b/results/dfa_pen_short/dfa_pen_lam0.0001_s42.json @@ -0,0 +1,43 @@ +{ + "config": { + "seed": 42, + "epochs": 30, + "lr": 0.001, + "wd": 0.01, + "lam": 0.0001, + "output_dir": "results/dfa_pen_short" + }, + "final_test_acc": 0.3593, + "log": [ + { + "epoch": 0, + "h_L_norm": 8.893179893493652, + "g_2_norm": 0.0009934091940522194, + "acc_eval": 0.115234375 + }, + { + "epoch": 1, + "h_L_norm": 3543.927490234375, + "g_2_norm": 2.68241001322167e-06, + "acc_eval": 0.3154296875 + }, + { + "epoch": 10, + "h_L_norm": 16542.669921875, + "g_2_norm": 7.705705229454907e-07, + "acc_eval": 0.349609375 + }, + { + "epoch": 20, + "h_L_norm": 22481.48046875, + "g_2_norm": 6.428867322938459e-07, + "acc_eval": 0.36328125 + }, + { + "epoch": 30, + "h_L_norm": 23648.146484375, + "g_2_norm": 6.275820396695053e-07, + "acc_eval": 0.3642578125 + } + ] +} \ No newline at end of file diff --git a/results/dfa_pen_short/dfa_pen_lam0.1_s42.json b/results/dfa_pen_short/dfa_pen_lam0.1_s42.json new file mode 100644 index 0000000..8364b09 --- /dev/null +++ b/results/dfa_pen_short/dfa_pen_lam0.1_s42.json @@ -0,0 +1,43 @@ +{ + "config": { + "seed": 42, + "epochs": 30, + "lr": 0.001, + "wd": 0.01, + "lam": 0.1, + "output_dir": "results/dfa_pen_short" + }, + "final_test_acc": 0.3486, + "log": [ + { + "epoch": 0, + "h_L_norm": 8.893179893493652, + "g_2_norm": 0.0009934091940522194, + "acc_eval": 0.115234375 + }, + { + "epoch": 1, + "h_L_norm": 731.34130859375, + "g_2_norm": 1.2336811778368428e-05, + "acc_eval": 0.3330078125 + }, + { + "epoch": 10, + "h_L_norm": 6931.87841796875, + "g_2_norm": 2.318319502592203e-06, + "acc_eval": 0.3466796875 + }, + { + "epoch": 20, + "h_L_norm": 10827.6552734375, + "g_2_norm": 1.7205680933329859e-06, + "acc_eval": 0.3564453125 + }, + { + "epoch": 30, + "h_L_norm": 11574.76171875, + "g_2_norm": 1.6344213236152427e-06, + "acc_eval": 0.359375 + } + ] +} \ No newline at end of file -- cgit v1.2.3