From 57aed9a01b4712a32c10a0cff980f7963d694bbf Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 05:23:14 -0500 Subject: results: close normalized C2 recovery --- RESULTS.md | 19 +++++++++++++++++++ ROADMAP.md | 7 +++++++ ..._nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s0.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s1.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s2.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s0.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s1.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s2.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s0.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s1.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s2.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s0.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s1.json | 1 + ..._nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s2.json | 1 + 14 files changed, 38 insertions(+) create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s0.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s1.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s2.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s0.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s1.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s2.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s0.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s1.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s2.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s0.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s1.json create mode 100644 results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s2.json diff --git a/RESULTS.md b/RESULTS.md index f7873a1..8ca07ad 100644 --- a/RESULTS.md +++ b/RESULTS.md @@ -446,6 +446,25 @@ neutral residualization raises the teaching squared cosine from `0.7492` to `0.9 task-period fitting lowers it to `0.5395` by removing soma-correlated teaching. **The frozen C5 theory package and numerical checks pass.** +### Normalized-regression development screen (task seed 0 only) + +The final bounded C2 recovery divided each context-vectorizer regression update by the complete +feature power of `[c, c outer tanh(h_top)]`. It fixed the feedback-first prefix at 100 layerwise-K4 +steps, retained simultaneous K1/e4 joint calibration, and compared only `eta_A in {0.01, 0.05}`. + +| NLMS eta_A | depth | validation acc (%) | depth gain | d4 lesion drop | d4 work / ordinary | +|-----------:|------:|-------------------:|-----------:|----------------:|-------------------:| +| 0.01 | 1 | 70.717 ± 2.759 | — | — | — | +| 0.01 | 4 | 75.150 ± 23.011 | +4.433 | +26.000 | 2.5x | +| 0.05 | 1 | 73.000 ± 2.783 | — | — | — | +| 0.05 | 4 | 73.350 ± 23.008 | +0.350 | +19.533 | 2.5x | + +Both settings have only 2/3 positive depth gains and lesions, and neither approaches the frozen +90% d4 threshold. Small per-example feature norms amplify perturbation noise despite finite final +losses. **No NLMS candidate is eligible; further tent-map tuning is closed and task seeds 6--8 +remain untouched.** Direct NP still establishes that causal local updates can solve the task, but +the current low-query amortized vectorizer cannot reliably track them. + ## Paper-faithful non-backprop baselines `PETITE` in the initial request is treated as **PEPITA**. Implementations and hyperparameters were diff --git a/ROADMAP.md b/ROADMAP.md index a28f9af..3d125b7 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -211,6 +211,13 @@ least 2/3 positive depth gains, and lesion mean at least 2 points with at least eligible candidates select the highest d4 mean, preferring `eta_A=0.01` within one point. Failure closes further tent-map tuning; success permits one frozen confirmation on task seeds 6--8. +**Normalized-regression status: failed development on 2026-07-22.** NLMS with `eta_A=0.01` and +`0.05` reached only `75.150 ± 23.011%` and `73.350 ± 23.008%` at d4, recovering mean depth gains +of `+4.433` and `+0.350` points. Each had only 2/3 positive gains and lesions. Both are ineligible; +the tent-map recovery program is closed, no task seed 6--8 result was generated, and C2 remains a +negative result for the amortized method. The passed direct-NP diagnosis and all failed recovery +mechanisms remain part of the evidence rather than being hidden by another search branch. + ### C3. Causal calibration is genuinely amortized **Status: passed on 2026-07-22.** The frozen five-seed confirmation retained 112.9% of the K16/e4 diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s0.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s0.json new file mode 100644 index 0000000..d0707d8 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 1, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.0, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.01, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s0", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.7099499106407166}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.6865, "eval_loss": 0.6074167785644531, "wall_s": 4.983481168746948, "val_acc": 0.6865, "val_loss": 0.6074167785644531, "cos_r_negg": [0.9996563196182251], "cos_innovation_negg": [0.9996563196182251], "cos_apical_negg": [0.9996563196182251], "cos_Ac_negg": [0.9996563196182251], "r_norm": [3.5343122482299805], "g_norm": [0.007498926017433405], "traffic_norm": [0.0], "traffic_residual_norm": [0.0], "traffic_r2": [NaN]}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 0.3911125659942627, "training_loop_wall_s": 4.525828123092651, "diagnostics_wall_s": 0.0439603328704834, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 4.525828123092651, "evaluation_wall_s": 0.01804041862487793}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 2180, "calibration_example_loss_evaluations": 552896, "calibration_forward_equivalent_examples": 650005.3333333334, "training_forward_equivalent_examples": 1315029.3333333335, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17401856, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s1.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s1.json new file mode 100644 index 0000000..b1f1382 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 1, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.0, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.01, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s1", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.6922826766967773}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.6965, "eval_loss": 0.6075302734375, "wall_s": 4.956945180892944, "val_acc": 0.6965, "val_loss": 0.6075302734375, "cos_r_negg": [0.9999009370803833], "cos_innovation_negg": [0.9999009370803833], "cos_apical_negg": [0.9999009370803833], "cos_Ac_negg": [0.9999009370803833], "r_norm": [3.601689100265503], "g_norm": [0.007350320927798748], "traffic_norm": [0.0], "traffic_residual_norm": [0.0], "traffic_r2": [NaN]}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 0.4044933319091797, "training_loop_wall_s": 4.483836889266968, "diagnostics_wall_s": 0.04549384117126465, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 4.483836889266968, "evaluation_wall_s": 0.018461942672729492}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 2180, "calibration_example_loss_evaluations": 552896, "calibration_forward_equivalent_examples": 650005.3333333334, "training_forward_equivalent_examples": 1315029.3333333335, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17401856, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s2.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s2.json new file mode 100644 index 0000000..bc64625 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 1, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.0, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.01, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d1_t0_s2", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.7055632472038269}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.7385, "eval_loss": 0.6114118041992187, "wall_s": 4.9738874435424805, "val_acc": 0.7385, "val_loss": 0.6114118041992187, "cos_r_negg": [0.9984827041625977], "cos_innovation_negg": [0.9984827041625977], "cos_apical_negg": [0.9984827041625977], "cos_Ac_negg": [0.9984827041625977], "r_norm": [3.0503664016723633], "g_norm": [0.006741282530128956], "traffic_norm": [0.0], "traffic_residual_norm": [0.0], "traffic_r2": [NaN]}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 0.38558125495910645, "training_loop_wall_s": 4.508875846862793, "diagnostics_wall_s": 0.056111812591552734, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 4.508875846862793, "evaluation_wall_s": 0.018304824829101562}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 2180, "calibration_example_loss_evaluations": 552896, "calibration_forward_equivalent_examples": 650005.3333333334, "training_forward_equivalent_examples": 1315029.3333333335, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17401856, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s0.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s0.json new file mode 100644 index 0000000..6575d2f --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 4, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.3333333333333333, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.01, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s0", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.6903698444366455}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.5, "eval_loss": 152828895.232, "wall_s": 14.316128730773926, "val_acc": 0.5, "val_loss": 152828895.232, "cos_r_negg": [NaN, 0.7278270125389099, NaN, NaN], "cos_innovation_negg": [NaN, 0.7278270125389099, NaN, NaN], "cos_apical_negg": [NaN, 0.7278270125389099, NaN, NaN], "cos_Ac_negg": [NaN, 0.7278270125389099, NaN, NaN], "r_norm": [2075400.5, 293532.90625, 579206.5, 603183.375], "g_norm": [136.99351501464844, 76.26994323730469, 76.26994323730469, 76.26994323730469], "traffic_norm": [0.0, 0.0, 0.0, 0.0], "traffic_residual_norm": [0.0, 0.0, 0.0, 0.0], "traffic_r2": [NaN, NaN, NaN, NaN], "residual_lesion": {"interior_layers": [1, 2, 3], "lesioned_layers": [3], "branch_to_skip_rms": [4.024543699638682e+16, 0.0, 0.0], "lesion_eval_acc": 0.5, "lesion_eval_loss": 152828895.232, "lesion_acc_drop": 0.0}}, "lesion_protocol": {"fraction_of_interior_blocks": 0.3333333333333333, "selection": "final_contiguous_interior_blocks", "probe_source": "training_prefix", "probe_examples": 512, "evaluation_split": "validation"}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 1.6722493171691895, "training_loop_wall_s": 12.538999319076538, "diagnostics_wall_s": 0.06386137008666992, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 12.538999319076538, "evaluation_wall_s": 0.03601861000061035}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 4580, "calibration_example_loss_evaluations": 1153472, "calibration_forward_equivalent_examples": 931757.0370370373, "training_forward_equivalent_examples": 1596781.0370370373, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17595904, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s1.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s1.json new file mode 100644 index 0000000..24be36a --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 4, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.3333333333333333, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.01, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s1", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.7494125366210938}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.9515, "eval_loss": 0.1741734390258789, "wall_s": 14.13081669807434, "val_acc": 0.9515, "val_loss": 0.1741734390258789, "cos_r_negg": [0.6063010692596436, 0.32109761238098145, 0.7849156856536865, 0.943676233291626], "cos_innovation_negg": [0.6063010692596436, 0.32109761238098145, 0.7849156856536865, 0.943676233291626], "cos_apical_negg": [0.6063010692596436, 0.32109761238098145, 0.7849156856536865, 0.943676233291626], "cos_Ac_negg": [0.6063010692596436, 0.32109761238098145, 0.7849156856536865, 0.943676233291626], "r_norm": [1.2907683849334717, 1.379010558128357, 1.0128116607666016, 0.5039305090904236], "g_norm": [0.00582353537902236, 0.0045771789737045765, 0.0025443341583013535, 0.0010863847564905882], "traffic_norm": [0.0, 0.0, 0.0, 0.0], "traffic_residual_norm": [0.0, 0.0, 0.0, 0.0], "traffic_r2": [NaN, NaN, NaN, NaN], "residual_lesion": {"interior_layers": [1, 2, 3], "lesioned_layers": [3], "branch_to_skip_rms": [0.9680498242378235, 1.2054117918014526, 1.3589704036712646], "lesion_eval_acc": 0.3655, "lesion_eval_loss": 5.793059814453125, "lesion_acc_drop": 0.5860000000000001}}, "lesion_protocol": {"fraction_of_interior_blocks": 0.3333333333333333, "selection": "final_contiguous_interior_blocks", "probe_source": "training_prefix", "probe_examples": 512, "evaluation_split": "validation"}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 1.3988580703735352, "training_loop_wall_s": 12.620917320251465, "diagnostics_wall_s": 0.07194757461547852, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 12.620917320251465, "evaluation_wall_s": 0.03405451774597168}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 4580, "calibration_example_loss_evaluations": 1153472, "calibration_forward_equivalent_examples": 931757.0370370373, "training_forward_equivalent_examples": 1596781.0370370373, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17595904, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s2.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s2.json new file mode 100644 index 0000000..c1d0977 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 4, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.3333333333333333, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.01, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p01_d4_t0_s2", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.6906555891036987}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.803, "eval_loss": 0.4907631225585937, "wall_s": 14.888797521591187, "val_acc": 0.803, "val_loss": 0.4907631225585937, "cos_r_negg": [-0.15610843896865845, 0.27225446701049805, -0.17267972230911255, 0.9923940896987915], "cos_innovation_negg": [-0.15610843896865845, 0.27225446701049805, -0.17267972230911255, 0.9923940896987915], "cos_apical_negg": [-0.15610843896865845, 0.27225446701049805, -0.17267972230911255, 0.9923940896987915], "cos_Ac_negg": [-0.15610843896865845, 0.27225446701049805, -0.17267972230911255, 0.9923940896987915], "r_norm": [1.4460375308990479, 1.5924363136291504, 1.3423223495483398, 2.6062376499176025], "g_norm": [0.0037894966080784798, 0.00399779062718153, 0.003136627608910203, 0.004482114687561989], "traffic_norm": [0.0, 0.0, 0.0, 0.0], "traffic_residual_norm": [0.0, 0.0, 0.0, 0.0], "traffic_r2": [NaN, NaN, NaN, NaN], "residual_lesion": {"interior_layers": [1, 2, 3], "lesioned_layers": [3], "branch_to_skip_rms": [0.44900938868522644, 0.6054736375808716, 0.9042813181877136], "lesion_eval_acc": 0.609, "lesion_eval_loss": 2.5926112060546873, "lesion_acc_drop": 0.19400000000000006}}, "lesion_protocol": {"fraction_of_interior_blocks": 0.3333333333333333, "selection": "final_contiguous_interior_blocks", "probe_source": "training_prefix", "probe_examples": 512, "evaluation_split": "validation"}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 1.4611456394195557, "training_loop_wall_s": 13.319961547851562, "diagnostics_wall_s": 0.06913948059082031, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 13.319961547851562, "evaluation_wall_s": 0.03344416618347168}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 4580, "calibration_example_loss_evaluations": 1153472, "calibration_forward_equivalent_examples": 931757.0370370373, "training_forward_equivalent_examples": 1596781.0370370373, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17595904, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s0.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s0.json new file mode 100644 index 0000000..f2b4fe7 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 1, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.0, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.05, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s0", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.7099499106407166}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.7325, "eval_loss": 0.5944062805175782, "wall_s": 5.1316869258880615, "val_acc": 0.7325, "val_loss": 0.5944062805175782, "cos_r_negg": [0.9997653365135193], "cos_innovation_negg": [0.9997653365135193], "cos_apical_negg": [0.9997653365135193], "cos_Ac_negg": [0.9997653365135193], "r_norm": [3.6750659942626953], "g_norm": [0.007048233412206173], "traffic_norm": [0.0], "traffic_residual_norm": [0.0], "traffic_r2": [NaN]}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 0.39478516578674316, "training_loop_wall_s": 4.661332607269287, "diagnostics_wall_s": 0.05255007743835449, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 4.661332607269287, "evaluation_wall_s": 0.018257856369018555}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 2180, "calibration_example_loss_evaluations": 552896, "calibration_forward_equivalent_examples": 650005.3333333334, "training_forward_equivalent_examples": 1315029.3333333335, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17401856, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s1.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s1.json new file mode 100644 index 0000000..37d3549 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 1, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.0, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.05, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s1", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.6922826766967773}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.7565, "eval_loss": 0.5970514526367188, "wall_s": 4.948259353637695, "val_acc": 0.7565, "val_loss": 0.5970514526367188, "cos_r_negg": [0.9999008178710938], "cos_innovation_negg": [0.9999008178710938], "cos_apical_negg": [0.9999008178710938], "cos_Ac_negg": [0.9999008178710938], "r_norm": [3.6053619384765625], "g_norm": [0.007007450331002474], "traffic_norm": [0.0], "traffic_residual_norm": [0.0], "traffic_r2": [NaN]}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 0.4134330749511719, "training_loop_wall_s": 4.460611581802368, "diagnostics_wall_s": 0.05065798759460449, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 4.460611581802368, "evaluation_wall_s": 0.018883466720581055}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 2180, "calibration_example_loss_evaluations": 552896, "calibration_forward_equivalent_examples": 650005.3333333334, "training_forward_equivalent_examples": 1315029.3333333335, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17401856, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s2.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s2.json new file mode 100644 index 0000000..402bc9a --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 1, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.0, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.05, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d1_t0_s2", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.7055632472038269}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.701, "eval_loss": 0.6080374450683593, "wall_s": 4.967407941818237, "val_acc": 0.701, "val_loss": 0.6080374450683593, "cos_r_negg": [0.9999068975448608], "cos_innovation_negg": [0.9999068975448608], "cos_apical_negg": [0.9999068975448608], "cos_Ac_negg": [0.9999068975448608], "r_norm": [3.480165481567383], "g_norm": [0.006693234667181969], "traffic_norm": [0.0], "traffic_residual_norm": [0.0], "traffic_r2": [NaN]}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 0.3776068687438965, "training_loop_wall_s": 4.51497483253479, "diagnostics_wall_s": 0.05142545700073242, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 4.51497483253479, "evaluation_wall_s": 0.018416404724121094}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 2180, "calibration_example_loss_evaluations": 552896, "calibration_forward_equivalent_examples": 650005.3333333334, "training_forward_equivalent_examples": 1315029.3333333335, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17401856, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s0.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s0.json new file mode 100644 index 0000000..5f02565 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 4, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.3333333333333333, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.05, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s0", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.6903698444366455}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.5, "eval_loss": 55.296529296875, "wall_s": 14.139602184295654, "val_acc": 0.5, "val_loss": 55.296529296875, "cos_r_negg": [-0.5886871814727783, -0.6601795554161072, NaN, -1.0], "cos_innovation_negg": [-0.5886871814727783, -0.6601795554161072, NaN, -1.0], "cos_apical_negg": [-0.5886871814727783, -0.6601795554161072, NaN, -1.0], "cos_Ac_negg": [-0.5886871814727783, -0.6601795554161072, NaN, -1.0], "r_norm": [2.3945956230163574, 2.1244421005249023, 2.060727119445801, 1.7774169445037842], "g_norm": [0.016381485387682915, 0.0055907610803842545, 0.0055907610803842545, 0.004795310087502003], "traffic_norm": [0.0, 0.0, 0.0, 0.0], "traffic_residual_norm": [0.0, 0.0, 0.0, 0.0], "traffic_r2": [NaN, NaN, NaN, NaN], "residual_lesion": {"interior_layers": [1, 2, 3], "lesioned_layers": [3], "branch_to_skip_rms": [5.011775493621826, 0.0, 2.7264139652252197], "lesion_eval_acc": 0.5, "lesion_eval_loss": 54.026951171875, "lesion_acc_drop": 0.0}}, "lesion_protocol": {"fraction_of_interior_blocks": 0.3333333333333333, "selection": "final_contiguous_interior_blocks", "probe_source": "training_prefix", "probe_examples": 512, "evaluation_split": "validation"}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 1.5171058177947998, "training_loop_wall_s": 12.49098801612854, "diagnostics_wall_s": 0.08894777297973633, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 12.49098801612854, "evaluation_wall_s": 0.03715777397155762}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 4580, "calibration_example_loss_evaluations": 1153472, "calibration_forward_equivalent_examples": 931757.0370370373, "training_forward_equivalent_examples": 1596781.0370370373, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17595904, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s1.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s1.json new file mode 100644 index 0000000..656e195 --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 4, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.3333333333333333, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.05, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s1", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.7494125366210938}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.96, "eval_loss": 0.09631807327270508, "wall_s": 13.635825634002686, "val_acc": 0.96, "val_loss": 0.09631807327270508, "cos_r_negg": [0.5974857807159424, 0.9998971223831177, 0.9935190677642822, 0.9614993333816528], "cos_innovation_negg": [0.5974857807159424, 0.9998971223831177, 0.9935190677642822, 0.9614993333816528], "cos_apical_negg": [0.5974857807159424, 0.9998971223831177, 0.9935190677642822, 0.9614993333816528], "cos_Ac_negg": [0.5974857807159424, 0.9998971223831177, 0.9935190677642822, 0.9614993333816528], "r_norm": [2.2339909076690674, 1.9335857629776, 1.376535415649414, 0.585070013999939], "g_norm": [0.004918329417705536, 0.003439245978370309, 0.0023866859264671803, 0.0010121234226971865], "traffic_norm": [0.0, 0.0, 0.0, 0.0], "traffic_residual_norm": [0.0, 0.0, 0.0, 0.0], "traffic_r2": [NaN, NaN, NaN, NaN], "residual_lesion": {"interior_layers": [1, 2, 3], "lesioned_layers": [3], "branch_to_skip_rms": [0.6693511605262756, 0.5386325120925903, 0.7986617684364319], "lesion_eval_acc": 0.6145, "lesion_eval_loss": 1.8886297607421876, "lesion_acc_drop": 0.3454999999999999}}, "lesion_protocol": {"fraction_of_interior_blocks": 0.3333333333333333, "selection": "final_contiguous_interior_blocks", "probe_source": "training_prefix", "probe_examples": 512, "evaluation_split": "validation"}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 1.3451290130615234, "training_loop_wall_s": 12.18733286857605, "diagnostics_wall_s": 0.06515073776245117, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 12.18733286857605, "evaluation_wall_s": 0.03352093696594238}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 4580, "calibration_example_loss_evaluations": 1153472, "calibration_forward_equivalent_examples": 931757.0370370373, "training_forward_equivalent_examples": 1596781.0370370373, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17595904, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file diff --git a/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s2.json b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s2.json new file mode 100644 index 0000000..9b36f9b --- /dev/null +++ b/results/c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "tentmap", "depth": 4, "width": 8, "act": "relu", "residual": 1, "residual_lesion_fraction": 0.3333333333333333, "epochs": 80, "batch_size": 256, "train_examples": 0, "val_examples": 2000, "split_seed": 2027, "eval_split": "validation", "task_seed": 0, "task_train_examples": 10000, "task_test_examples": 5000, "task_levels": 2, "task_n_in": 1, "task_classes": 10, "teacher_depth": 8, "teacher_width": 64, "teacher_residual": 1, "eta": 0.03, "eta_A": 0.05, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "feedback_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 1, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "a_warmup_steps": 100, "a_warmup_ndirs": 4, "a_warmup_mode": "layerwise", "nuis_rho": 0.0, "traffic_seed": 1234, "traffic_mode": "none", "predictor_mode": "diagonal", "vectorizer_mode": "context_gated", "vectorizer_optimizer": "nlms", "vectorizer_eps": 1e-06, "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100000, "diagnostics": "alignment", "diagnostics_schedule": "final", "eval_every": 0, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "c2_nlmsdev_v1_tent_l2_w8_context_e0p05_d4_t0_s2", "n_in": 1, "n_out": 2}, "split": {"dataset": "tentmap", "task_seed": 0, "train_examples": 8000, "test_examples": 5000, "evaluation_split": "validation", "synthetic_generator": true, "split_seed": 2027, "validation_examples": 2000, "validation_index_sha256": "f292d9bfeef1b6e79c80d4cf1c572ac12dee1be799a71a126390709d0386c6e4", "validation_class_counts": {"0": 1000, "1": 1000}, "split_from_training_only": true}, "provenance": {"git_commit": "533fd8e7877350bda29384f780abae8bf3ead12a", "git_dirty": false}, "diagnostic_protocol": {"probe_source": "training_prefix", "probe_examples": 512, "schedule": "final"}, "steps": [{"step": 0, "epoch": 0, "train_loss": 0.6906555891036987}, {"epoch_end": 0, "step": 32}, {"epoch_end": 1, "step": 64}, {"epoch_end": 2, "step": 96}, {"epoch_end": 3, "step": 128}, {"epoch_end": 4, "step": 160}, {"epoch_end": 5, "step": 192}, {"epoch_end": 6, "step": 224}, {"epoch_end": 7, "step": 256}, {"epoch_end": 8, "step": 288}, {"epoch_end": 9, "step": 320}, {"epoch_end": 10, "step": 352}, {"epoch_end": 11, "step": 384}, {"epoch_end": 12, "step": 416}, {"epoch_end": 13, "step": 448}, {"epoch_end": 14, "step": 480}, {"epoch_end": 15, "step": 512}, {"epoch_end": 16, "step": 544}, {"epoch_end": 17, "step": 576}, {"epoch_end": 18, "step": 608}, {"epoch_end": 19, "step": 640}, {"epoch_end": 20, "step": 672}, {"epoch_end": 21, "step": 704}, {"epoch_end": 22, "step": 736}, {"epoch_end": 23, "step": 768}, {"epoch_end": 24, "step": 800}, {"epoch_end": 25, "step": 832}, {"epoch_end": 26, "step": 864}, {"epoch_end": 27, "step": 896}, {"epoch_end": 28, "step": 928}, {"epoch_end": 29, "step": 960}, {"epoch_end": 30, "step": 992}, {"epoch_end": 31, "step": 1024}, {"epoch_end": 32, "step": 1056}, {"epoch_end": 33, "step": 1088}, {"epoch_end": 34, "step": 1120}, {"epoch_end": 35, "step": 1152}, {"epoch_end": 36, "step": 1184}, {"epoch_end": 37, "step": 1216}, {"epoch_end": 38, "step": 1248}, {"epoch_end": 39, "step": 1280}, {"epoch_end": 40, "step": 1312}, {"epoch_end": 41, "step": 1344}, {"epoch_end": 42, "step": 1376}, {"epoch_end": 43, "step": 1408}, {"epoch_end": 44, "step": 1440}, {"epoch_end": 45, "step": 1472}, {"epoch_end": 46, "step": 1504}, {"epoch_end": 47, "step": 1536}, {"epoch_end": 48, "step": 1568}, {"epoch_end": 49, "step": 1600}, {"epoch_end": 50, "step": 1632}, {"epoch_end": 51, "step": 1664}, {"epoch_end": 52, "step": 1696}, {"epoch_end": 53, "step": 1728}, {"epoch_end": 54, "step": 1760}, {"epoch_end": 55, "step": 1792}, {"epoch_end": 56, "step": 1824}, {"epoch_end": 57, "step": 1856}, {"epoch_end": 58, "step": 1888}, {"epoch_end": 59, "step": 1920}, {"epoch_end": 60, "step": 1952}, {"epoch_end": 61, "step": 1984}, {"epoch_end": 62, "step": 2016}, {"epoch_end": 63, "step": 2048}, {"epoch_end": 64, "step": 2080}, {"epoch_end": 65, "step": 2112}, {"epoch_end": 66, "step": 2144}, {"epoch_end": 67, "step": 2176}, {"epoch_end": 68, "step": 2208}, {"epoch_end": 69, "step": 2240}, {"epoch_end": 70, "step": 2272}, {"epoch_end": 71, "step": 2304}, {"epoch_end": 72, "step": 2336}, {"epoch_end": 73, "step": 2368}, {"epoch_end": 74, "step": 2400}, {"epoch_end": 75, "step": 2432}, {"epoch_end": 76, "step": 2464}, {"epoch_end": 77, "step": 2496}, {"epoch_end": 78, "step": 2528}, {"epoch_end": 79, "step": 2560}], "final": {"eval_split": "validation", "eval_acc": 0.7405, "eval_loss": 0.6226087341308594, "wall_s": 13.959492683410645, "val_acc": 0.7405, "val_loss": 0.6226087341308594, "cos_r_negg": [0.33318161964416504, 0.13834625482559204, 0.9050644636154175, 0.9948923587799072], "cos_innovation_negg": [0.33318161964416504, 0.13834625482559204, 0.9050644636154175, 0.9948923587799072], "cos_apical_negg": [0.33318161964416504, 0.13834625482559204, 0.9050644636154175, 0.9948923587799072], "cos_Ac_negg": [0.33318161964416504, 0.13834625482559204, 0.9050644636154175, 0.9948923587799072], "r_norm": [1.6418747901916504, 2.0537407398223877, 2.0100455284118652, 1.625048279762268], "g_norm": [0.0038734516128897667, 0.00371169694699347, 0.003319336799904704, 0.0030191908590495586], "traffic_norm": [0.0, 0.0, 0.0, 0.0], "traffic_residual_norm": [0.0, 0.0, 0.0, 0.0], "traffic_r2": [NaN, NaN, NaN, NaN], "residual_lesion": {"interior_layers": [1, 2, 3], "lesioned_layers": [3], "branch_to_skip_rms": [1.0634486675262451, 0.8184682726860046, 1.2348827123641968], "lesion_eval_acc": 0.5, "lesion_eval_loss": 3.5326451416015625, "lesion_acc_drop": 0.24050000000000005}}, "lesion_protocol": {"fraction_of_interior_blocks": 0.3333333333333333, "selection": "final_contiguous_interior_blocks", "probe_source": "training_prefix", "probe_examples": 512, "evaluation_split": "validation"}, "timing": {"warmup_wall_s": 0.0, "feedback_warmup_wall_s": 1.351172685623169, "training_loop_wall_s": 12.50026535987854, "diagnostics_wall_s": 0.06874990463256836, "inline_diagnostics_wall_s": 0.0, "optimizer_wall_s_excluding_inline_diagnostics": 12.50026535987854, "evaluation_wall_s": 0.03416323661804199}, "cost": {"train_steps": 2560, "ordinary_training_forward_examples": 640000, "predictor_warmup_forward_examples": 0, "feedback_warmup_forward_examples": 25024, "feedback_warmup_perturbation_events": 100, "perturbation_events": 740, "calibration_batch_loss_evaluations": 4580, "calibration_example_loss_evaluations": 1153472, "calibration_forward_equivalent_examples": 931757.0370370373, "training_forward_equivalent_examples": 1596781.0370370373, "max_perturbation_batch_expansion": 2}, "hardware": {"device": "cuda", "torch_version": "2.3.1+cu118", "cuda_device_name": "NVIDIA GeForce GTX 1080", "cuda_visible_devices": "7", "device_total_memory_bytes": 8507949056, "peak_memory_allocated_bytes": 17595904, "peak_memory_reserved_bytes": 23068672}} \ No newline at end of file -- cgit v1.2.3