1 files changed, 31 insertions, 0 deletions
diff --git a/research/flossing/late_perturb_robustness/trm_multi4_final_step65100_n3000_k8_late.summary.csv b/research/flossing/late_perturb_robustness/trm_multi4_final_step65100_n3000_k8_late.summary.csv
new file mode 100644
index 0000000..b4d80d6
--- /dev/null
+++ b/research/flossing/late_perturb_robustness/trm_multi4_final_step65100_n3000_k8_late.summary.csv
@@ -0,0 +1,31 @@
+label,perturb_after,sigma,n_samples,rollouts,ckpt_root,ckpt_name,perturb,noise_distribution,mean_rollout_exact,mean_rollout_token_acc,pass_at_k,all_k,correct_count_mean,correct_count_std,zero_frac,full_frac,clean_acc,retain_mean_on_clean_success,allK_on_clean_success,rescue_mean_on_clean_fail,passK_on_clean_fail
+trm_multi4_final,0,0.0,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8286666870117188,0.9328684210777283,0.8286666870117188,0.8286666870117188,6.62933349609375,3.014399766921997,0.17133332788944244,0.8286666870117188,0.8286666870117188,1.0,1.0,0.0,0.0
+trm_multi4_final,0,0.001,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8319166898727417,0.9341394901275635,0.9086666703224182,0.7463333606719971,6.655333518981934,2.654029369354248,0.09133332967758179,0.7463333606719971,0.8286666870117188,0.9579645991325378,0.8942075371742249,0.22227627038955688,0.4902723729610443
+trm_multi4_final,0,0.003,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8315416574478149,0.93404221534729,0.9143333435058594,0.7419999837875366,6.6523332595825195,2.6464054584503174,0.08566666394472122,0.7419999837875366,0.8286666870117188,0.9564561247825623,0.8897827863693237,0.22738327085971832,0.5252918004989624
+trm_multi4_final,0,0.01,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8301666378974915,0.9333322644233704,0.9143333435058594,0.7356666922569275,6.641333103179932,2.639954090118408,0.08566666394472122,0.7356666922569275,0.8286666870117188,0.9536906480789185,0.879324197769165,0.23273345828056335,0.5272373557090759
+trm_multi4_final,0,0.03,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8319166898727417,0.934055507183075,0.9213333129882812,0.7239999771118164,6.655333518981934,2.594970464706421,0.07866666465997696,0.7239999771118164,0.8286666870117188,0.9528358578681946,0.8688656687736511,0.2470817118883133,0.5661478638648987
+trm_multi4_final,0,0.1,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8354583382606506,0.9354007244110107,0.9269999861717224,0.7246666550636292,6.683666706085205,2.5519142150878906,0.0729999989271164,0.7246666550636292,0.8286666870117188,0.9477574229240417,0.861625075340271,0.29231518507003784,0.5992217659950256
+trm_multi4_final,4,0.0,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8286666870117188,0.9328684210777283,0.8286666870117188,0.8286666870117188,6.62933349609375,3.014399766921997,0.17133332788944244,0.8286666870117188,0.8286666870117188,1.0,1.0,0.0,0.0
+trm_multi4_final,4,0.001,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8335833549499512,0.9348888397216797,0.8993333578109741,0.7756666541099548,6.668666839599609,2.732560157775879,0.10066666454076767,0.7756666541099548,0.8286666870117188,0.970585286617279,0.9336283206939697,0.17096303403377533,0.43968871235847473
+trm_multi4_final,4,0.003,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8352500200271606,0.9354547262191772,0.9020000100135803,0.7749999761581421,6.682000160217285,2.7150585651397705,0.09799999743700027,0.7749999761581421,0.8286666870117188,0.971238911151886,0.9328238368034363,0.17752918601036072,0.4494163393974304
+trm_multi4_final,4,0.01,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8327916860580444,0.9343785643577576,0.8989999890327454,0.7720000147819519,6.6623334884643555,2.7240254878997803,0.10100000351667404,0.7720000147819519,0.8286666870117188,0.9690265655517578,0.9292035102844238,0.17388132214546204,0.4319066107273102
+trm_multi4_final,4,0.03,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8316666483879089,0.9340242743492126,0.8986666798591614,0.7730000019073486,6.6533331871032715,2.7423510551452637,0.10133333504199982,0.7730000019073486,0.8286666870117188,0.9677695035934448,0.9267899990081787,0.17339494824409485,0.4319066107273102
+trm_multi4_final,4,0.1,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8324583172798157,0.9344562888145447,0.8989999890327454,0.7680000066757202,6.659666538238525,2.7232038974761963,0.10100000351667404,0.7680000066757202,0.8286666870117188,0.9691271185874939,0.9239742755889893,0.1714494228363037,0.4319066107273102
+trm_multi4_final,8,0.0,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8286666870117188,0.9328684210777283,0.8286666870117188,0.8286666870117188,6.62933349609375,3.014399766921997,0.17133332788944244,0.8286666870117188,0.8286666870117188,1.0,1.0,0.0,0.0
+trm_multi4_final,8,0.001,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8332499861717224,0.934550940990448,0.8849999904632568,0.8013333082199097,6.665999889373779,2.8155124187469482,0.11500000208616257,0.8013333082199097,0.8286666870117188,0.9811444282531738,0.9662107825279236,0.11794747412204742,0.344357967376709
+trm_multi4_final,8,0.003,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8323749899864197,0.9341363906860352,0.8830000162124634,0.8016666769981384,6.658999919891357,2.8219470977783203,0.11699999868869781,0.8016666769981384,0.8286666870117188,0.9805410504341125,0.967015266418457,0.1157587543129921,0.3346303403377533
+trm_multi4_final,8,0.01,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8352916836738586,0.9354629516601562,0.8856666684150696,0.8016666769981384,6.682333469390869,2.7959651947021484,0.11433333158493042,0.8016666769981384,0.8286666870117188,0.9822506308555603,0.9666130542755127,0.12451361864805222,0.344357967376709
+trm_multi4_final,8,0.03,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8335000276565552,0.9347721338272095,0.890333354473114,0.7993333339691162,6.668000221252441,2.803647994995117,0.10966666787862778,0.7993333339691162,0.8286666870117188,0.9798873662948608,0.9625905156135559,0.1254863739013672,0.3696497976779938
+trm_multi4_final,8,0.1,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8331666588783264,0.9347808957099915,0.8916666507720947,0.7960000038146973,6.665333271026611,2.7949953079223633,0.10833333432674408,0.7960000038146973,0.8286666870117188,0.9781777858734131,0.9593724608421326,0.13180933892726898,0.38910505175590515
+trm_multi4_final,12,0.0,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8286666870117188,0.9328684210777283,0.8286666870117188,0.8286666870117188,6.62933349609375,3.014399766921997,0.17133332788944244,0.8286666870117188,0.8286666870117188,1.0,1.0,0.0,0.0
+trm_multi4_final,12,0.001,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8307916522026062,0.9338425993919373,0.8569999933242798,0.8209999799728394,6.64633321762085,2.936537742614746,0.14300000667572021,0.8209999799728394,0.8286666870117188,0.9936142563819885,0.9907481670379639,0.043287936598062515,0.1750972718000412
+trm_multi4_final,12,0.003,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8307499885559082,0.9337176084518433,0.8603333234786987,0.8209999799728394,6.645999908447266,2.9328060150146484,0.13966666162014008,0.8209999799728394,0.8286666870117188,0.9935137033462524,0.9907481670379639,0.04353112727403641,0.18871594965457916
+trm_multi4_final,12,0.01,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8307499885559082,0.9339007139205933,0.8583333492279053,0.8203333616256714,6.645999908447266,2.93530535697937,0.14166666567325592,0.8203333616256714,0.8286666870117188,0.993463397026062,0.9899436831474304,0.0437743179500103,0.17898832261562347
+trm_multi4_final,12,0.03,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.831166684627533,0.9338744282722473,0.8616666793823242,0.8190000057220459,6.649333477020264,2.921477794647217,0.13833333551883698,0.8190000057220459,0.8286666870117188,0.9930108785629272,0.9883346557617188,0.04839494079351425,0.20428015291690826
+trm_multi4_final,12,0.1,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8309583067893982,0.9341686964035034,0.8683333396911621,0.8149999976158142,6.6476664543151855,2.9015274047851562,0.1316666603088379,0.8149999976158142,0.8286666870117188,0.9908487796783447,0.9835076332092285,0.057636186480522156,0.24319066107273102
+trm_multi4_final,15,0.0,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8286666870117188,0.9328684210777283,0.8286666870117188,0.8286666870117188,6.62933349609375,3.014399766921997,0.17133332788944244,0.8286666870117188,0.8286666870117188,1.0,1.0,0.0,0.0
+trm_multi4_final,15,0.001,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8286666870117188,0.9330503940582275,0.8286666870117188,0.8286666870117188,6.62933349609375,3.014399766921997,0.17133332788944244,0.8286666870117188,0.8286666870117188,1.0,1.0,0.0,0.0
+trm_multi4_final,15,0.003,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8287083506584167,0.9331271052360535,0.8289999961853027,0.8286666870117188,6.629666805267334,3.0137219429016113,0.17100000381469727,0.8286666870117188,0.8286666870117188,1.0,1.0,0.00024319066142197698,0.0019455252913758159
+trm_multi4_final,15,0.01,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8286250233650208,0.9330225586891174,0.8289999961853027,0.828000009059906,6.629000186920166,3.0135293006896973,0.17100000381469727,0.828000009059906,0.8286666870117188,0.9998994469642639,0.9991955161094666,0.00024319066142197698,0.0019455252913758159
+trm_multi4_final,15,0.03,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8287500143051147,0.9331192970275879,0.8306666612625122,0.8276666402816772,6.630000114440918,3.009169340133667,0.1693333387374878,0.8276666402816772,0.8286666870117188,0.999698281288147,0.9987932443618774,0.0019455252913758159,0.011673151515424252
+trm_multi4_final,15,0.1,3000,8,/home/yurenh2/rrm/trm/checkpoints/Sudoku-extreme-1k-aug-1000-ACT-torch/pretrain_mlp_t_sudoku_official_gbs768_multi4_loguniform_repro,step_65100,both,gaussian,0.8292916417121887,0.9331137537956238,0.8330000042915344,0.8276666402816772,6.63433313369751,3.0008811950683594,0.16699999570846558,0.8276666402816772,0.8286666870117188,0.9995977282524109,0.9987932443618774,0.0055933850817382336,0.02529182843863964