From 4d0186956cc3d1adda1d7937d9efdde2507fdf6d Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Tue, 21 Jul 2026 08:53:51 -0500 Subject: results: record audited scaling and baseline runs --- results/baseline_budget_v1_mnist_ff_w256_d3_s0.json | 1 + results/baseline_budget_v1_mnist_ff_w256_d3_s1.json | 1 + results/baseline_budget_v1_mnist_ff_w256_d3_s2.json | 1 + results/baseline_budget_v1_mnist_ff_w256_d3_s3.json | 1 + results/baseline_budget_v1_mnist_ff_w256_d3_s4.json | 1 + results/ep_archmatch_v1_mnist_bp_w500_d1_s0.json | 1 + results/ep_archmatch_v1_mnist_bp_w500_d1_s1.json | 1 + results/ep_archmatch_v1_mnist_dfa_w500_d1_s0.json | 1 + results/ep_archmatch_v1_mnist_dfa_w500_d1_s1.json | 1 + results/ep_archmatch_v1_mnist_sdil_w500_d1_s0.json | 1 + results/ep_archmatch_v1_mnist_sdil_w500_d1_s1.json | 1 + results/ep_match_v1_mnist_bp_w256_d3_s0.json | 1 + results/ep_match_v1_mnist_bp_w256_d3_s1.json | 1 + results/ep_match_v1_mnist_dfa_w256_d3_s0.json | 1 + results/ep_match_v1_mnist_dfa_w256_d3_s1.json | 1 + results/ep_match_v1_mnist_sdil_w256_d3_s0.json | 1 + results/ep_match_v1_mnist_sdil_w256_d3_s1.json | 1 + results/ep_original_v2_mnist_ep_w500_d1_s0.json | 1 + results/ep_original_v2_mnist_ep_w500_d1_s1.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d30_s0.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d30_s1.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d30_s2.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d30_s3.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d30_s4.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d60_s0.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d60_s1.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d60_s2.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d60_s3.json | 1 + results/fa_scale_v2_cifar10_fa_w64_d60_s4.json | 1 + results/nuis_ctrl_v1_rho0_matched_s0.json | 1 + results/nuis_ctrl_v1_rho0_matched_s1.json | 1 + results/nuis_ctrl_v1_rho0_matched_s2.json | 1 + results/nuis_ctrl_v1_rho0_matched_s3.json | 1 + results/nuis_ctrl_v1_rho0_matched_s4.json | 1 + results/nuis_ctrl_v1_rho0_raw_s0.json | 1 + results/nuis_ctrl_v1_rho0_raw_s1.json | 1 + results/nuis_ctrl_v1_rho0_raw_s2.json | 1 + results/nuis_ctrl_v1_rho0_raw_s3.json | 1 + results/nuis_ctrl_v1_rho0_raw_s4.json | 1 + results/nuis_ctrl_v1_rho0_residual_s0.json | 1 + results/nuis_ctrl_v1_rho0_residual_s1.json | 1 + results/nuis_ctrl_v1_rho0_residual_s2.json | 1 + results/nuis_ctrl_v1_rho0_residual_s3.json | 1 + results/nuis_ctrl_v1_rho0_residual_s4.json | 1 + results/nuis_ctrl_v1_rho0p05_matched_s0.json | 1 + results/nuis_ctrl_v1_rho0p05_matched_s1.json | 1 + results/nuis_ctrl_v1_rho0p05_matched_s2.json | 1 + results/nuis_ctrl_v1_rho0p05_matched_s3.json | 1 + results/nuis_ctrl_v1_rho0p05_matched_s4.json | 1 + results/nuis_ctrl_v1_rho0p05_raw_s0.json | 1 + results/nuis_ctrl_v1_rho0p05_raw_s1.json | 1 + results/nuis_ctrl_v1_rho0p05_raw_s2.json | 1 + results/nuis_ctrl_v1_rho0p05_raw_s3.json | 1 + results/nuis_ctrl_v1_rho0p05_raw_s4.json | 1 + results/nuis_ctrl_v1_rho0p05_residual_s0.json | 1 + results/nuis_ctrl_v1_rho0p05_residual_s1.json | 1 + results/nuis_ctrl_v1_rho0p05_residual_s2.json | 1 + results/nuis_ctrl_v1_rho0p05_residual_s3.json | 1 + results/nuis_ctrl_v1_rho0p05_residual_s4.json | 1 + results/nuis_ctrl_v1_rho0p2_matched_s0.json | 1 + results/nuis_ctrl_v1_rho0p2_matched_s1.json | 1 + results/nuis_ctrl_v1_rho0p2_matched_s2.json | 1 + results/nuis_ctrl_v1_rho0p2_matched_s3.json | 1 + results/nuis_ctrl_v1_rho0p2_matched_s4.json | 1 + results/nuis_ctrl_v1_rho0p2_raw_s0.json | 1 + results/nuis_ctrl_v1_rho0p2_raw_s1.json | 1 + results/nuis_ctrl_v1_rho0p2_raw_s2.json | 1 + results/nuis_ctrl_v1_rho0p2_raw_s3.json | 1 + results/nuis_ctrl_v1_rho0p2_raw_s4.json | 1 + results/nuis_ctrl_v1_rho0p2_residual_s0.json | 1 + results/nuis_ctrl_v1_rho0p2_residual_s1.json | 1 + results/nuis_ctrl_v1_rho0p2_residual_s2.json | 1 + results/nuis_ctrl_v1_rho0p2_residual_s3.json | 1 + results/nuis_ctrl_v1_rho0p2_residual_s4.json | 1 + results/nuis_ctrl_v1_rho0p5_matched_s0.json | 1 + results/nuis_ctrl_v1_rho0p5_matched_s1.json | 1 + results/nuis_ctrl_v1_rho0p5_matched_s2.json | 1 + results/nuis_ctrl_v1_rho0p5_matched_s3.json | 1 + results/nuis_ctrl_v1_rho0p5_matched_s4.json | 1 + results/nuis_ctrl_v1_rho0p5_raw_s0.json | 1 + results/nuis_ctrl_v1_rho0p5_raw_s1.json | 1 + results/nuis_ctrl_v1_rho0p5_raw_s2.json | 1 + results/nuis_ctrl_v1_rho0p5_raw_s3.json | 1 + results/nuis_ctrl_v1_rho0p5_raw_s4.json | 1 + results/nuis_ctrl_v1_rho0p5_residual_s0.json | 1 + results/nuis_ctrl_v1_rho0p5_residual_s1.json | 1 + results/nuis_ctrl_v1_rho0p5_residual_s2.json | 1 + results/nuis_ctrl_v1_rho0p5_residual_s3.json | 1 + results/nuis_ctrl_v1_rho0p5_residual_s4.json | 1 + results/pepita_tuned_v1_mnist_pepita_w256_d3_s0.json | 1 + results/pepita_tuned_v1_mnist_pepita_w256_d3_s1.json | 1 + results/pepita_tuned_v1_mnist_pepita_w256_d3_s2.json | 1 + results/pepita_tuned_v1_mnist_pepita_w256_d3_s3.json | 1 + results/pepita_tuned_v1_mnist_pepita_w256_d3_s4.json | 1 + results/scale_v3_cifar10_bp_w64_d30_s0.json | 1 + results/scale_v3_cifar10_bp_w64_d30_s1.json | 1 + results/scale_v3_cifar10_bp_w64_d30_s2.json | 1 + results/scale_v3_cifar10_bp_w64_d30_s3.json | 1 + results/scale_v3_cifar10_bp_w64_d30_s4.json | 1 + results/scale_v3_cifar10_bp_w64_d60_s0.json | 1 + results/scale_v3_cifar10_bp_w64_d60_s1.json | 1 + results/scale_v3_cifar10_bp_w64_d60_s2.json | 1 + results/scale_v3_cifar10_bp_w64_d60_s3.json | 1 + results/scale_v3_cifar10_bp_w64_d60_s4.json | 1 + results/scale_v3_cifar10_dfa_w64_d30_s0.json | 1 + results/scale_v3_cifar10_dfa_w64_d30_s1.json | 1 + results/scale_v3_cifar10_dfa_w64_d30_s2.json | 1 + results/scale_v3_cifar10_dfa_w64_d30_s3.json | 1 + results/scale_v3_cifar10_dfa_w64_d30_s4.json | 1 + results/scale_v3_cifar10_dfa_w64_d60_s0.json | 1 + results/scale_v3_cifar10_dfa_w64_d60_s1.json | 1 + results/scale_v3_cifar10_dfa_w64_d60_s2.json | 1 + results/scale_v3_cifar10_dfa_w64_d60_s3.json | 1 + results/scale_v3_cifar10_dfa_w64_d60_s4.json | 1 + results/scale_v3_cifar10_sdil_w64_d30_s0.json | 1 + results/scale_v3_cifar10_sdil_w64_d30_s1.json | 1 + results/scale_v3_cifar10_sdil_w64_d30_s2.json | 1 + results/scale_v3_cifar10_sdil_w64_d30_s3.json | 1 + results/scale_v3_cifar10_sdil_w64_d30_s4.json | 1 + results/scale_v3_cifar10_sdil_w64_d60_s0.json | 1 + results/scale_v3_cifar10_sdil_w64_d60_s1.json | 1 + results/scale_v3_cifar10_sdil_w64_d60_s2.json | 1 + results/scale_v3_cifar10_sdil_w64_d60_s3.json | 1 + results/scale_v3_cifar10_sdil_w64_d60_s4.json | 1 + 124 files changed, 124 insertions(+) create mode 100644 results/baseline_budget_v1_mnist_ff_w256_d3_s0.json create mode 100644 results/baseline_budget_v1_mnist_ff_w256_d3_s1.json create mode 100644 results/baseline_budget_v1_mnist_ff_w256_d3_s2.json create mode 100644 results/baseline_budget_v1_mnist_ff_w256_d3_s3.json create mode 100644 results/baseline_budget_v1_mnist_ff_w256_d3_s4.json create mode 100644 results/ep_archmatch_v1_mnist_bp_w500_d1_s0.json create mode 100644 results/ep_archmatch_v1_mnist_bp_w500_d1_s1.json create mode 100644 results/ep_archmatch_v1_mnist_dfa_w500_d1_s0.json create mode 100644 results/ep_archmatch_v1_mnist_dfa_w500_d1_s1.json create mode 100644 results/ep_archmatch_v1_mnist_sdil_w500_d1_s0.json create mode 100644 results/ep_archmatch_v1_mnist_sdil_w500_d1_s1.json create mode 100644 results/ep_match_v1_mnist_bp_w256_d3_s0.json create mode 100644 results/ep_match_v1_mnist_bp_w256_d3_s1.json create mode 100644 results/ep_match_v1_mnist_dfa_w256_d3_s0.json create mode 100644 results/ep_match_v1_mnist_dfa_w256_d3_s1.json create mode 100644 results/ep_match_v1_mnist_sdil_w256_d3_s0.json create mode 100644 results/ep_match_v1_mnist_sdil_w256_d3_s1.json create mode 100644 results/ep_original_v2_mnist_ep_w500_d1_s0.json create mode 100644 results/ep_original_v2_mnist_ep_w500_d1_s1.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d30_s0.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d30_s1.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d30_s2.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d30_s3.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d30_s4.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d60_s0.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d60_s1.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d60_s2.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d60_s3.json create mode 100644 results/fa_scale_v2_cifar10_fa_w64_d60_s4.json create mode 100644 results/nuis_ctrl_v1_rho0_matched_s0.json create mode 100644 results/nuis_ctrl_v1_rho0_matched_s1.json create mode 100644 results/nuis_ctrl_v1_rho0_matched_s2.json create mode 100644 results/nuis_ctrl_v1_rho0_matched_s3.json create mode 100644 results/nuis_ctrl_v1_rho0_matched_s4.json create mode 100644 results/nuis_ctrl_v1_rho0_raw_s0.json create mode 100644 results/nuis_ctrl_v1_rho0_raw_s1.json create mode 100644 results/nuis_ctrl_v1_rho0_raw_s2.json create mode 100644 results/nuis_ctrl_v1_rho0_raw_s3.json create mode 100644 results/nuis_ctrl_v1_rho0_raw_s4.json create mode 100644 results/nuis_ctrl_v1_rho0_residual_s0.json create mode 100644 results/nuis_ctrl_v1_rho0_residual_s1.json create mode 100644 results/nuis_ctrl_v1_rho0_residual_s2.json create mode 100644 results/nuis_ctrl_v1_rho0_residual_s3.json create mode 100644 results/nuis_ctrl_v1_rho0_residual_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p05_matched_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p05_matched_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p05_matched_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p05_matched_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p05_matched_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p05_raw_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p05_raw_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p05_raw_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p05_raw_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p05_raw_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p05_residual_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p05_residual_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p05_residual_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p05_residual_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p05_residual_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p2_matched_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p2_matched_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p2_matched_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p2_matched_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p2_matched_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p2_raw_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p2_raw_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p2_raw_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p2_raw_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p2_raw_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p2_residual_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p2_residual_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p2_residual_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p2_residual_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p2_residual_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p5_matched_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p5_matched_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p5_matched_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p5_matched_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p5_matched_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p5_raw_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p5_raw_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p5_raw_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p5_raw_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p5_raw_s4.json create mode 100644 results/nuis_ctrl_v1_rho0p5_residual_s0.json create mode 100644 results/nuis_ctrl_v1_rho0p5_residual_s1.json create mode 100644 results/nuis_ctrl_v1_rho0p5_residual_s2.json create mode 100644 results/nuis_ctrl_v1_rho0p5_residual_s3.json create mode 100644 results/nuis_ctrl_v1_rho0p5_residual_s4.json create mode 100644 results/pepita_tuned_v1_mnist_pepita_w256_d3_s0.json create mode 100644 results/pepita_tuned_v1_mnist_pepita_w256_d3_s1.json create mode 100644 results/pepita_tuned_v1_mnist_pepita_w256_d3_s2.json create mode 100644 results/pepita_tuned_v1_mnist_pepita_w256_d3_s3.json create mode 100644 results/pepita_tuned_v1_mnist_pepita_w256_d3_s4.json create mode 100644 results/scale_v3_cifar10_bp_w64_d30_s0.json create mode 100644 results/scale_v3_cifar10_bp_w64_d30_s1.json create mode 100644 results/scale_v3_cifar10_bp_w64_d30_s2.json create mode 100644 results/scale_v3_cifar10_bp_w64_d30_s3.json create mode 100644 results/scale_v3_cifar10_bp_w64_d30_s4.json create mode 100644 results/scale_v3_cifar10_bp_w64_d60_s0.json create mode 100644 results/scale_v3_cifar10_bp_w64_d60_s1.json create mode 100644 results/scale_v3_cifar10_bp_w64_d60_s2.json create mode 100644 results/scale_v3_cifar10_bp_w64_d60_s3.json create mode 100644 results/scale_v3_cifar10_bp_w64_d60_s4.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d30_s0.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d30_s1.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d30_s2.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d30_s3.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d30_s4.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d60_s0.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d60_s1.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d60_s2.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d60_s3.json create mode 100644 results/scale_v3_cifar10_dfa_w64_d60_s4.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d30_s0.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d30_s1.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d30_s2.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d30_s3.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d30_s4.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d60_s0.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d60_s1.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d60_s2.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d60_s3.json create mode 100644 results/scale_v3_cifar10_sdil_w64_d60_s4.json diff --git a/results/baseline_budget_v1_mnist_ff_w256_d3_s0.json b/results/baseline_budget_v1_mnist_ff_w256_d3_s0.json new file mode 100644 index 0000000..306c98b --- /dev/null +++ b/results/baseline_budget_v1_mnist_ff_w256_d3_s0.json @@ -0,0 +1 @@ +{"args": {"method": "ff", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 15, "batch_size": 128, "max_batches": 0, "eta": null, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 0, "device": "cuda", "outdir": "results", "tag": "baseline_budget_v1_mnist_ff_w256_d3_s0"}, "method_source": {"paper": "https://www.cs.toronto.edu/~hinton/absps/FFXfinal.pdf", "reference": "https://github.com/mpezeshki/pytorch_forward_forward", "protocol": "greedy layerwise goodness; input length normalization; Adam"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"layer_end": 0, "test_acc": 0.0797, "local_loss": 0.24380753934383392}, {"layer_end": 1, "test_acc": 0.9395, "local_loss": 0.2586314082145691}, {"layer_end": 2, "test_acc": 0.9391, "local_loss": 0.30240654945373535}], "final": {"test_acc": 0.9391, "test_loss": NaN, "wall_s": 67.42461061477661}} \ No newline at end of file diff --git a/results/baseline_budget_v1_mnist_ff_w256_d3_s1.json b/results/baseline_budget_v1_mnist_ff_w256_d3_s1.json new file mode 100644 index 0000000..7f17f81 --- /dev/null +++ b/results/baseline_budget_v1_mnist_ff_w256_d3_s1.json @@ -0,0 +1 @@ +{"args": {"method": "ff", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 15, "batch_size": 128, "max_batches": 0, "eta": null, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 1, "device": "cuda", "outdir": "results", "tag": "baseline_budget_v1_mnist_ff_w256_d3_s1"}, "method_source": {"paper": "https://www.cs.toronto.edu/~hinton/absps/FFXfinal.pdf", "reference": "https://github.com/mpezeshki/pytorch_forward_forward", "protocol": "greedy layerwise goodness; input length normalization; Adam"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"layer_end": 0, "test_acc": 0.0723, "local_loss": 0.2413138449192047}, {"layer_end": 1, "test_acc": 0.9384, "local_loss": 0.26700910925865173}, {"layer_end": 2, "test_acc": 0.9345, "local_loss": 0.3376375138759613}], "final": {"test_acc": 0.9345, "test_loss": NaN, "wall_s": 68.828449010849}} \ No newline at end of file diff --git a/results/baseline_budget_v1_mnist_ff_w256_d3_s2.json b/results/baseline_budget_v1_mnist_ff_w256_d3_s2.json new file mode 100644 index 0000000..327c0d7 --- /dev/null +++ b/results/baseline_budget_v1_mnist_ff_w256_d3_s2.json @@ -0,0 +1 @@ +{"args": {"method": "ff", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 15, "batch_size": 128, "max_batches": 0, "eta": null, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 2, "device": "cuda", "outdir": "results", "tag": "baseline_budget_v1_mnist_ff_w256_d3_s2"}, "method_source": {"paper": "https://www.cs.toronto.edu/~hinton/absps/FFXfinal.pdf", "reference": "https://github.com/mpezeshki/pytorch_forward_forward", "protocol": "greedy layerwise goodness; input length normalization; Adam"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"layer_end": 0, "test_acc": 0.1426, "local_loss": 0.2255902737379074}, {"layer_end": 1, "test_acc": 0.9362, "local_loss": 0.2764553427696228}, {"layer_end": 2, "test_acc": 0.9334, "local_loss": 0.3405337333679199}], "final": {"test_acc": 0.9334, "test_loss": NaN, "wall_s": 68.85102558135986}} \ No newline at end of file diff --git a/results/baseline_budget_v1_mnist_ff_w256_d3_s3.json b/results/baseline_budget_v1_mnist_ff_w256_d3_s3.json new file mode 100644 index 0000000..c5e4f7f --- /dev/null +++ b/results/baseline_budget_v1_mnist_ff_w256_d3_s3.json @@ -0,0 +1 @@ +{"args": {"method": "ff", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 15, "batch_size": 128, "max_batches": 0, "eta": null, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 3, "device": "cuda", "outdir": "results", "tag": "baseline_budget_v1_mnist_ff_w256_d3_s3"}, "method_source": {"paper": "https://www.cs.toronto.edu/~hinton/absps/FFXfinal.pdf", "reference": "https://github.com/mpezeshki/pytorch_forward_forward", "protocol": "greedy layerwise goodness; input length normalization; Adam"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"layer_end": 0, "test_acc": 0.1039, "local_loss": 0.24703803658485413}, {"layer_end": 1, "test_acc": 0.9392, "local_loss": 0.24402467906475067}, {"layer_end": 2, "test_acc": 0.9367, "local_loss": 0.2870331406593323}], "final": {"test_acc": 0.9367, "test_loss": NaN, "wall_s": 67.10431051254272}} \ No newline at end of file diff --git a/results/baseline_budget_v1_mnist_ff_w256_d3_s4.json b/results/baseline_budget_v1_mnist_ff_w256_d3_s4.json new file mode 100644 index 0000000..47c2755 --- /dev/null +++ b/results/baseline_budget_v1_mnist_ff_w256_d3_s4.json @@ -0,0 +1 @@ +{"args": {"method": "ff", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 15, "batch_size": 128, "max_batches": 0, "eta": null, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 4, "device": "cuda", "outdir": "results", "tag": "baseline_budget_v1_mnist_ff_w256_d3_s4"}, "method_source": {"paper": "https://www.cs.toronto.edu/~hinton/absps/FFXfinal.pdf", "reference": "https://github.com/mpezeshki/pytorch_forward_forward", "protocol": "greedy layerwise goodness; input length normalization; Adam"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"layer_end": 0, "test_acc": 0.0941, "local_loss": 0.22019338607788086}, {"layer_end": 1, "test_acc": 0.9345, "local_loss": 0.2702571749687195}, {"layer_end": 2, "test_acc": 0.9342, "local_loss": 0.2818516492843628}], "final": {"test_acc": 0.9342, "test_loss": NaN, "wall_s": 68.65620350837708}} \ No newline at end of file diff --git a/results/ep_archmatch_v1_mnist_bp_w500_d1_s0.json b/results/ep_archmatch_v1_mnist_bp_w500_d1_s0.json new file mode 100644 index 0000000..b76f8ad --- /dev/null +++ b/results/ep_archmatch_v1_mnist_bp_w500_d1_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "mnist", "depth": 1, "width": 500, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_archmatch_v1_mnist_bp_w500_d1_s0", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.529653310775757}, {"step": 50, "epoch": 0, "train_loss": 0.3679444491863251}, {"step": 100, "epoch": 0, "train_loss": 0.13640330731868744}, {"step": 150, "epoch": 0, "train_loss": 0.17397169768810272}, {"step": 200, "epoch": 0, "train_loss": 0.22615139186382294}, {"step": 250, "epoch": 0, "train_loss": 0.2479219138622284}, {"step": 300, "epoch": 0, "train_loss": 0.13611888885498047}, {"step": 350, "epoch": 0, "train_loss": 0.20692390203475952}, {"epoch_end": 0, "step": 391, "test_acc": 0.9589, "test_loss": 0.14059993324279785}, {"step": 400, "epoch": 1, "train_loss": 0.09336370974779129}, {"step": 450, "epoch": 1, "train_loss": 0.12225135415792465}, {"step": 500, "epoch": 1, "train_loss": 0.18298286199569702}, {"step": 550, "epoch": 1, "train_loss": 0.06902388483285904}, {"step": 600, "epoch": 1, "train_loss": 0.05321333557367325}, {"step": 650, "epoch": 1, "train_loss": 0.06309623271226883}, {"step": 700, "epoch": 1, "train_loss": 0.07216529548168182}, {"step": 750, "epoch": 1, "train_loss": 0.11980332434177399}, {"epoch_end": 1, "step": 782, "test_acc": 0.9579, "test_loss": 0.14021187896728515}, {"step": 800, "epoch": 2, "train_loss": 0.06986840069293976}, {"step": 850, "epoch": 2, "train_loss": 0.030502747744321823}, {"step": 900, "epoch": 2, "train_loss": 0.056895650923252106}, {"step": 950, "epoch": 2, "train_loss": 0.08159191906452179}, {"step": 1000, "epoch": 2, "train_loss": 0.036765407770872116}, {"step": 1050, "epoch": 2, "train_loss": 0.02545097842812538}, {"step": 1100, "epoch": 2, "train_loss": 0.1005951389670372}, {"step": 1150, "epoch": 2, "train_loss": 0.06085049733519554}, {"epoch_end": 2, "step": 1173, "test_acc": 0.973, "test_loss": 0.08655920886993408}, {"step": 1200, "epoch": 3, "train_loss": 0.04218049719929695}, {"step": 1250, "epoch": 3, "train_loss": 0.03542773425579071}, {"step": 1300, "epoch": 3, "train_loss": 0.013723227195441723}, {"step": 1350, "epoch": 3, "train_loss": 0.01605125702917576}, {"step": 1400, "epoch": 3, "train_loss": 0.039044760167598724}, {"step": 1450, "epoch": 3, "train_loss": 0.021958449855446815}, {"step": 1500, "epoch": 3, "train_loss": 0.07918249815702438}, {"step": 1550, "epoch": 3, "train_loss": 0.012839997187256813}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9744, "test_loss": 0.08105616683959961}, {"step": 1600, "epoch": 4, "train_loss": 0.017535462975502014}, {"step": 1650, "epoch": 4, "train_loss": 0.0428149588406086}, {"step": 1700, "epoch": 4, "train_loss": 0.03784516826272011}, {"step": 1750, "epoch": 4, "train_loss": 0.011868010275065899}, {"step": 1800, "epoch": 4, "train_loss": 0.013396226800978184}, {"step": 1850, "epoch": 4, "train_loss": 0.030906831845641136}, {"step": 1900, "epoch": 4, "train_loss": 0.004803402815014124}, {"step": 1950, "epoch": 4, "train_loss": 0.03004857711493969}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9784, "test_loss": 0.06740753145217895}, {"step": 2000, "epoch": 5, "train_loss": 0.01953575201332569}, {"step": 2050, "epoch": 5, "train_loss": 0.0072065661661326885}, {"step": 2100, "epoch": 5, "train_loss": 0.0072195082902908325}, {"step": 2150, "epoch": 5, "train_loss": 0.004397705662995577}, {"step": 2200, "epoch": 5, "train_loss": 0.0043753632344305515}, {"step": 2250, "epoch": 5, "train_loss": 0.011778188869357109}, {"step": 2300, "epoch": 5, "train_loss": 0.0061593614518642426}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9807, "test_loss": 0.06374471864700318}, {"step": 2350, "epoch": 6, "train_loss": 0.008232607506215572}, {"step": 2400, "epoch": 6, "train_loss": 0.00517288688570261}, {"step": 2450, "epoch": 6, "train_loss": 0.004978879354894161}, {"step": 2500, "epoch": 6, "train_loss": 0.005841410718858242}, {"step": 2550, "epoch": 6, "train_loss": 0.005197310820221901}, {"step": 2600, "epoch": 6, "train_loss": 0.007580540608614683}, {"step": 2650, "epoch": 6, "train_loss": 0.0027406231965869665}, {"step": 2700, "epoch": 6, "train_loss": 0.0035414493177086115}, {"epoch_end": 6, "step": 2737, "test_acc": 0.981, "test_loss": 0.06266844778060914}, {"step": 2750, "epoch": 7, "train_loss": 0.0012068471405655146}, {"step": 2800, "epoch": 7, "train_loss": 0.002066554967314005}, {"step": 2850, "epoch": 7, "train_loss": 0.0018961241003125906}, {"step": 2900, "epoch": 7, "train_loss": 0.0038532742764800787}, {"step": 2950, "epoch": 7, "train_loss": 0.0012177226599305868}, {"step": 3000, "epoch": 7, "train_loss": 0.0017440242227166891}, {"step": 3050, "epoch": 7, "train_loss": 0.004497661255300045}, {"step": 3100, "epoch": 7, "train_loss": 0.0014345506206154823}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9808, "test_loss": 0.06407360563278199}, {"step": 3150, "epoch": 8, "train_loss": 0.0021706335246562958}, {"step": 3200, "epoch": 8, "train_loss": 0.004896128550171852}, {"step": 3250, "epoch": 8, "train_loss": 0.0015725011471658945}, {"step": 3300, "epoch": 8, "train_loss": 0.002179396105930209}, {"step": 3350, "epoch": 8, "train_loss": 0.004903758876025677}, {"step": 3400, "epoch": 8, "train_loss": 0.0024478230625391006}, {"step": 3450, "epoch": 8, "train_loss": 0.003042479744181037}, {"step": 3500, "epoch": 8, "train_loss": 0.004097072873264551}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9809, "test_loss": 0.062189706611633304}, {"step": 3550, "epoch": 9, "train_loss": 0.0025155050680041313}, {"step": 3600, "epoch": 9, "train_loss": 0.0018959719454869628}, {"step": 3650, "epoch": 9, "train_loss": 0.0028564154636114836}, {"step": 3700, "epoch": 9, "train_loss": 0.003433483885601163}, {"step": 3750, "epoch": 9, "train_loss": 0.003114477964118123}, {"step": 3800, "epoch": 9, "train_loss": 0.0013581393286585808}, {"step": 3850, "epoch": 9, "train_loss": 0.002901146188378334}, {"step": 3900, "epoch": 9, "train_loss": 0.0016000766772776842}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9815, "test_loss": 0.06230254135131836}, {"step": 3950, "epoch": 10, "train_loss": 0.001686303410679102}, {"step": 4000, "epoch": 10, "train_loss": 0.0020305109210312366}, {"step": 4050, "epoch": 10, "train_loss": 0.001804674044251442}, {"step": 4100, "epoch": 10, "train_loss": 0.0021467830520123243}, {"step": 4150, "epoch": 10, "train_loss": 0.001022701384499669}, {"step": 4200, "epoch": 10, "train_loss": 0.00189292267896235}, {"step": 4250, "epoch": 10, "train_loss": 0.002707919105887413}, {"step": 4300, "epoch": 10, "train_loss": 0.0020615390967577696}, {"epoch_end": 10, "step": 4301, "test_acc": 0.981, "test_loss": 0.06365604772567748}, {"step": 4350, "epoch": 11, "train_loss": 0.001540582743473351}, {"step": 4400, "epoch": 11, "train_loss": 0.0011863976251333952}, {"step": 4450, "epoch": 11, "train_loss": 0.0021892886143177748}, {"step": 4500, "epoch": 11, "train_loss": 0.0004710734647233039}, {"step": 4550, "epoch": 11, "train_loss": 0.002431733999401331}, {"step": 4600, "epoch": 11, "train_loss": 0.00120242137927562}, {"step": 4650, "epoch": 11, "train_loss": 0.0024771057069301605}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9815, "test_loss": 0.06353156914710999}, {"step": 4700, "epoch": 12, "train_loss": 0.0018547872314229608}, {"step": 4750, "epoch": 12, "train_loss": 0.0015066111227497458}, {"step": 4800, "epoch": 12, "train_loss": 0.0011405418626964092}, {"step": 4850, "epoch": 12, "train_loss": 0.001623340300284326}, {"step": 4900, "epoch": 12, "train_loss": 0.0011404147371649742}, {"step": 4950, "epoch": 12, "train_loss": 0.0010571741731837392}, {"step": 5000, "epoch": 12, "train_loss": 0.0010574639309197664}, {"step": 5050, "epoch": 12, "train_loss": 0.0016814677510410547}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9813, "test_loss": 0.06383639097213745}, {"step": 5100, "epoch": 13, "train_loss": 0.0012780990218743682}, {"step": 5150, "epoch": 13, "train_loss": 0.0011164924362674356}, {"step": 5200, "epoch": 13, "train_loss": 0.0013468394754454494}, {"step": 5250, "epoch": 13, "train_loss": 0.0012687569251284003}, {"step": 5300, "epoch": 13, "train_loss": 0.0012173623545095325}, {"step": 5350, "epoch": 13, "train_loss": 0.002035332378000021}, {"step": 5400, "epoch": 13, "train_loss": 0.0008949877228587866}, {"step": 5450, "epoch": 13, "train_loss": 0.0009048383217304945}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9807, "test_loss": 0.06414769086837768}, {"step": 5500, "epoch": 14, "train_loss": 0.0007456267485395074}, {"step": 5550, "epoch": 14, "train_loss": 0.000963466358371079}, {"step": 5600, "epoch": 14, "train_loss": 0.0014049309538677335}, {"step": 5650, "epoch": 14, "train_loss": 0.0010383682092651725}, {"step": 5700, "epoch": 14, "train_loss": 0.0005266761872917414}, {"step": 5750, "epoch": 14, "train_loss": 0.0013183809351176023}, {"step": 5800, "epoch": 14, "train_loss": 0.0018872757209464908}, {"step": 5850, "epoch": 14, "train_loss": 0.001406779047101736}, {"epoch_end": 14, "step": 5865, "test_acc": 0.981, "test_loss": 0.06410251598358155}, {"step": 5900, "epoch": 15, "train_loss": 0.0010196907678619027}, {"step": 5950, "epoch": 15, "train_loss": 0.0016147011192515492}, {"step": 6000, "epoch": 15, "train_loss": 0.0021771234460175037}, {"step": 6050, "epoch": 15, "train_loss": 0.0018777959048748016}, {"step": 6100, "epoch": 15, "train_loss": 0.0012702962849289179}, {"step": 6150, "epoch": 15, "train_loss": 0.0009211473516188562}, {"step": 6200, "epoch": 15, "train_loss": 0.001036160159856081}, {"step": 6250, "epoch": 15, "train_loss": 0.0011221254244446754}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9818, "test_loss": 0.06406165990829468}, {"step": 6300, "epoch": 16, "train_loss": 0.001057866495102644}, {"step": 6350, "epoch": 16, "train_loss": 0.0007410273537971079}, {"step": 6400, "epoch": 16, "train_loss": 0.0007802165346220136}, {"step": 6450, "epoch": 16, "train_loss": 0.0005045710131525993}, {"step": 6500, "epoch": 16, "train_loss": 0.0007480365457013249}, {"step": 6550, "epoch": 16, "train_loss": 0.0012487847125157714}, {"step": 6600, "epoch": 16, "train_loss": 0.0007416029693558812}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9818, "test_loss": 0.0649446382522583}, {"step": 6650, "epoch": 17, "train_loss": 0.0005390990991145372}, {"step": 6700, "epoch": 17, "train_loss": 0.0003866490151267499}, {"step": 6750, "epoch": 17, "train_loss": 0.001493216142989695}, {"step": 6800, "epoch": 17, "train_loss": 0.0014812815934419632}, {"step": 6850, "epoch": 17, "train_loss": 0.0008846998098306358}, {"step": 6900, "epoch": 17, "train_loss": 0.001293561072088778}, {"step": 6950, "epoch": 17, "train_loss": 0.0010452351998537779}, {"step": 7000, "epoch": 17, "train_loss": 0.0011768215335905552}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9811, "test_loss": 0.06530593686103821}, {"step": 7050, "epoch": 18, "train_loss": 0.0003853194648399949}, {"step": 7100, "epoch": 18, "train_loss": 0.0014464458217844367}, {"step": 7150, "epoch": 18, "train_loss": 0.0004936479381285608}, {"step": 7200, "epoch": 18, "train_loss": 0.0007469477714039385}, {"step": 7250, "epoch": 18, "train_loss": 0.001263497630134225}, {"step": 7300, "epoch": 18, "train_loss": 0.0006984496139921248}, {"step": 7350, "epoch": 18, "train_loss": 0.0008992237271741033}, {"step": 7400, "epoch": 18, "train_loss": 0.0005116224638186395}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9819, "test_loss": 0.06473410873413087}, {"step": 7450, "epoch": 19, "train_loss": 0.0004476758767850697}, {"step": 7500, "epoch": 19, "train_loss": 0.000766622310038656}, {"step": 7550, "epoch": 19, "train_loss": 0.00045481123379431665}, {"step": 7600, "epoch": 19, "train_loss": 0.00035759585443884134}, {"step": 7650, "epoch": 19, "train_loss": 0.0011191722005605698}, {"step": 7700, "epoch": 19, "train_loss": 0.0010361210443079472}, {"step": 7750, "epoch": 19, "train_loss": 0.000923388113733381}, {"step": 7800, "epoch": 19, "train_loss": 0.001387335709296167}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9814, "test_loss": 0.06546387457847595}, {"step": 7850, "epoch": 20, "train_loss": 0.0008525121374987066}, {"step": 7900, "epoch": 20, "train_loss": 0.0007822598563507199}, {"step": 7950, "epoch": 20, "train_loss": 0.0007487335824407637}, {"step": 8000, "epoch": 20, "train_loss": 0.0007169679738581181}, {"step": 8050, "epoch": 20, "train_loss": 0.001149622374214232}, {"step": 8100, "epoch": 20, "train_loss": 0.0009398512775078416}, {"step": 8150, "epoch": 20, "train_loss": 0.0008019875385798514}, {"step": 8200, "epoch": 20, "train_loss": 0.0004135927592869848}, {"epoch_end": 20, "step": 8211, "test_acc": 0.982, "test_loss": 0.06540369753837585}, {"step": 8250, "epoch": 21, "train_loss": 0.0008482952835038304}, {"step": 8300, "epoch": 21, "train_loss": 0.0008039422682486475}, {"step": 8350, "epoch": 21, "train_loss": 0.0009830057388171554}, {"step": 8400, "epoch": 21, "train_loss": 0.0005507324240170419}, {"step": 8450, "epoch": 21, "train_loss": 0.0007066864054650068}, {"step": 8500, "epoch": 21, "train_loss": 0.000986541504971683}, {"step": 8550, "epoch": 21, "train_loss": 0.0006164516671560705}, {"step": 8600, "epoch": 21, "train_loss": 0.0012085899943485856}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9815, "test_loss": 0.06610920982360839}, {"step": 8650, "epoch": 22, "train_loss": 0.0005312811117619276}, {"step": 8700, "epoch": 22, "train_loss": 0.0006736447685398161}, {"step": 8750, "epoch": 22, "train_loss": 0.0006585121736861765}, {"step": 8800, "epoch": 22, "train_loss": 0.0009882417507469654}, {"step": 8850, "epoch": 22, "train_loss": 0.0005559821147471666}, {"step": 8900, "epoch": 22, "train_loss": 0.0013187630102038383}, {"step": 8950, "epoch": 22, "train_loss": 0.0006247776909731328}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9818, "test_loss": 0.06566711745262147}, {"step": 9000, "epoch": 23, "train_loss": 0.0005404988769441843}, {"step": 9050, "epoch": 23, "train_loss": 0.000696440227329731}, {"step": 9100, "epoch": 23, "train_loss": 0.0006092993426136672}, {"step": 9150, "epoch": 23, "train_loss": 0.0010911618592217565}, {"step": 9200, "epoch": 23, "train_loss": 0.0009640745120123029}, {"step": 9250, "epoch": 23, "train_loss": 0.0006281345267780125}, {"step": 9300, "epoch": 23, "train_loss": 0.0006386042223311961}, {"step": 9350, "epoch": 23, "train_loss": 0.0007003872306086123}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9821, "test_loss": 0.06649206714630126}, {"step": 9400, "epoch": 24, "train_loss": 0.0007319796714000404}, {"step": 9450, "epoch": 24, "train_loss": 0.0004183913697488606}, {"step": 9500, "epoch": 24, "train_loss": 0.0007594765047542751}, {"step": 9550, "epoch": 24, "train_loss": 0.0005724520306102931}, {"step": 9600, "epoch": 24, "train_loss": 0.0005377248744480312}, {"step": 9650, "epoch": 24, "train_loss": 0.0011918899836018682}, {"step": 9700, "epoch": 24, "train_loss": 0.0005883764824829996}, {"step": 9750, "epoch": 24, "train_loss": 0.0011136219836771488}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9818, "test_loss": 0.06645027012825012}], "final": {"test_acc": 0.9818, "test_loss": 0.06645027012825012, "wall_s": 15.702399015426636}} \ No newline at end of file diff --git a/results/ep_archmatch_v1_mnist_bp_w500_d1_s1.json b/results/ep_archmatch_v1_mnist_bp_w500_d1_s1.json new file mode 100644 index 0000000..3892ae2 --- /dev/null +++ b/results/ep_archmatch_v1_mnist_bp_w500_d1_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "mnist", "depth": 1, "width": 500, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_archmatch_v1_mnist_bp_w500_d1_s1", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.5871329307556152}, {"step": 50, "epoch": 0, "train_loss": 0.39890098571777344}, {"step": 100, "epoch": 0, "train_loss": 0.1446666717529297}, {"step": 150, "epoch": 0, "train_loss": 0.20340964198112488}, {"step": 200, "epoch": 0, "train_loss": 0.24126137793064117}, {"step": 250, "epoch": 0, "train_loss": 0.2480408400297165}, {"step": 300, "epoch": 0, "train_loss": 0.12762364745140076}, {"step": 350, "epoch": 0, "train_loss": 0.21582037210464478}, {"epoch_end": 0, "step": 391, "test_acc": 0.9588, "test_loss": 0.13509607696533202}, {"step": 400, "epoch": 1, "train_loss": 0.10601303726434708}, {"step": 450, "epoch": 1, "train_loss": 0.09629080444574356}, {"step": 500, "epoch": 1, "train_loss": 0.16499464213848114}, {"step": 550, "epoch": 1, "train_loss": 0.056739404797554016}, {"step": 600, "epoch": 1, "train_loss": 0.051818735897541046}, {"step": 650, "epoch": 1, "train_loss": 0.04696515575051308}, {"step": 700, "epoch": 1, "train_loss": 0.07741166651248932}, {"step": 750, "epoch": 1, "train_loss": 0.10981205850839615}, {"epoch_end": 1, "step": 782, "test_acc": 0.9573, "test_loss": 0.13575042343139648}, {"step": 800, "epoch": 2, "train_loss": 0.044692862778902054}, {"step": 850, "epoch": 2, "train_loss": 0.03525165840983391}, {"step": 900, "epoch": 2, "train_loss": 0.050680652260780334}, {"step": 950, "epoch": 2, "train_loss": 0.04522700235247612}, {"step": 1000, "epoch": 2, "train_loss": 0.048619482666254044}, {"step": 1050, "epoch": 2, "train_loss": 0.029327550902962685}, {"step": 1100, "epoch": 2, "train_loss": 0.08091145753860474}, {"step": 1150, "epoch": 2, "train_loss": 0.06784474849700928}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9741, "test_loss": 0.08735835494995117}, {"step": 1200, "epoch": 3, "train_loss": 0.0165726151317358}, {"step": 1250, "epoch": 3, "train_loss": 0.04066908732056618}, {"step": 1300, "epoch": 3, "train_loss": 0.008179138414561749}, {"step": 1350, "epoch": 3, "train_loss": 0.015870926901698112}, {"step": 1400, "epoch": 3, "train_loss": 0.03164156898856163}, {"step": 1450, "epoch": 3, "train_loss": 0.02011047676205635}, {"step": 1500, "epoch": 3, "train_loss": 0.053256843239068985}, {"step": 1550, "epoch": 3, "train_loss": 0.010040141642093658}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9753, "test_loss": 0.08263243579864502}, {"step": 1600, "epoch": 4, "train_loss": 0.020867835730314255}, {"step": 1650, "epoch": 4, "train_loss": 0.04469379782676697}, {"step": 1700, "epoch": 4, "train_loss": 0.04280044138431549}, {"step": 1750, "epoch": 4, "train_loss": 0.010804235003888607}, {"step": 1800, "epoch": 4, "train_loss": 0.012454817071557045}, {"step": 1850, "epoch": 4, "train_loss": 0.029494017362594604}, {"step": 1900, "epoch": 4, "train_loss": 0.011972269974648952}, {"step": 1950, "epoch": 4, "train_loss": 0.02718648500740528}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9795, "test_loss": 0.06712198448181152}, {"step": 2000, "epoch": 5, "train_loss": 0.013826706446707249}, {"step": 2050, "epoch": 5, "train_loss": 0.008596048690378666}, {"step": 2100, "epoch": 5, "train_loss": 0.004607761278748512}, {"step": 2150, "epoch": 5, "train_loss": 0.004207622725516558}, {"step": 2200, "epoch": 5, "train_loss": 0.0061744023114442825}, {"step": 2250, "epoch": 5, "train_loss": 0.006513251923024654}, {"step": 2300, "epoch": 5, "train_loss": 0.006806463934481144}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9801, "test_loss": 0.06427488451004028}, {"step": 2350, "epoch": 6, "train_loss": 0.005186669062823057}, {"step": 2400, "epoch": 6, "train_loss": 0.0022477556485682726}, {"step": 2450, "epoch": 6, "train_loss": 0.003183370688930154}, {"step": 2500, "epoch": 6, "train_loss": 0.005781187675893307}, {"step": 2550, "epoch": 6, "train_loss": 0.007169949822127819}, {"step": 2600, "epoch": 6, "train_loss": 0.00462106429040432}, {"step": 2650, "epoch": 6, "train_loss": 0.003371710190549493}, {"step": 2700, "epoch": 6, "train_loss": 0.00296182744204998}, {"epoch_end": 6, "step": 2737, "test_acc": 0.981, "test_loss": 0.06130636186599731}, {"step": 2750, "epoch": 7, "train_loss": 0.0008065736037679017}, {"step": 2800, "epoch": 7, "train_loss": 0.00151254388038069}, {"step": 2850, "epoch": 7, "train_loss": 0.002290093805640936}, {"step": 2900, "epoch": 7, "train_loss": 0.003927832003682852}, {"step": 2950, "epoch": 7, "train_loss": 0.0016205140855163336}, {"step": 3000, "epoch": 7, "train_loss": 0.0025558373890817165}, {"step": 3050, "epoch": 7, "train_loss": 0.006718539632856846}, {"step": 3100, "epoch": 7, "train_loss": 0.0014756751479580998}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9809, "test_loss": 0.06284126148223877}, {"step": 3150, "epoch": 8, "train_loss": 0.002305472269654274}, {"step": 3200, "epoch": 8, "train_loss": 0.0036424915306270123}, {"step": 3250, "epoch": 8, "train_loss": 0.001662460621446371}, {"step": 3300, "epoch": 8, "train_loss": 0.0023970715701580048}, {"step": 3350, "epoch": 8, "train_loss": 0.006178963463753462}, {"step": 3400, "epoch": 8, "train_loss": 0.002414938760921359}, {"step": 3450, "epoch": 8, "train_loss": 0.003943121526390314}, {"step": 3500, "epoch": 8, "train_loss": 0.004470392595976591}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9812, "test_loss": 0.06141401448249817}, {"step": 3550, "epoch": 9, "train_loss": 0.002596326172351837}, {"step": 3600, "epoch": 9, "train_loss": 0.0020087093580514193}, {"step": 3650, "epoch": 9, "train_loss": 0.0025333689991384745}, {"step": 3700, "epoch": 9, "train_loss": 0.002647845074534416}, {"step": 3750, "epoch": 9, "train_loss": 0.002976495074108243}, {"step": 3800, "epoch": 9, "train_loss": 0.0014495400246232748}, {"step": 3850, "epoch": 9, "train_loss": 0.0022415004204958677}, {"step": 3900, "epoch": 9, "train_loss": 0.0016766763292253017}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9814, "test_loss": 0.060905590105056764}, {"step": 3950, "epoch": 10, "train_loss": 0.0015614725416526198}, {"step": 4000, "epoch": 10, "train_loss": 0.0019684263970702887}, {"step": 4050, "epoch": 10, "train_loss": 0.0010777284624055028}, {"step": 4100, "epoch": 10, "train_loss": 0.001977615524083376}, {"step": 4150, "epoch": 10, "train_loss": 0.0007489322451874614}, {"step": 4200, "epoch": 10, "train_loss": 0.0019777691923081875}, {"step": 4250, "epoch": 10, "train_loss": 0.0031016438733786345}, {"step": 4300, "epoch": 10, "train_loss": 0.002164656762033701}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9811, "test_loss": 0.06269298119544983}, {"step": 4350, "epoch": 11, "train_loss": 0.001981984358280897}, {"step": 4400, "epoch": 11, "train_loss": 0.0012199783232063055}, {"step": 4450, "epoch": 11, "train_loss": 0.002503744326531887}, {"step": 4500, "epoch": 11, "train_loss": 0.00035784090869128704}, {"step": 4550, "epoch": 11, "train_loss": 0.0024510554503649473}, {"step": 4600, "epoch": 11, "train_loss": 0.0014480010140687227}, {"step": 4650, "epoch": 11, "train_loss": 0.0019454946741461754}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9819, "test_loss": 0.062189513969421385}, {"step": 4700, "epoch": 12, "train_loss": 0.0023562407586723566}, {"step": 4750, "epoch": 12, "train_loss": 0.0017537267412990332}, {"step": 4800, "epoch": 12, "train_loss": 0.0009359689429402351}, {"step": 4850, "epoch": 12, "train_loss": 0.0021071636583656073}, {"step": 4900, "epoch": 12, "train_loss": 0.0009586447849869728}, {"step": 4950, "epoch": 12, "train_loss": 0.0013441243208944798}, {"step": 5000, "epoch": 12, "train_loss": 0.001103094546124339}, {"step": 5050, "epoch": 12, "train_loss": 0.0012083410983905196}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9817, "test_loss": 0.06313391494750976}, {"step": 5100, "epoch": 13, "train_loss": 0.0012947010109201074}, {"step": 5150, "epoch": 13, "train_loss": 0.001314098946750164}, {"step": 5200, "epoch": 13, "train_loss": 0.0009943112963810563}, {"step": 5250, "epoch": 13, "train_loss": 0.0009577485034242272}, {"step": 5300, "epoch": 13, "train_loss": 0.0010958467610180378}, {"step": 5350, "epoch": 13, "train_loss": 0.0014758426696062088}, {"step": 5400, "epoch": 13, "train_loss": 0.0011308171087875962}, {"step": 5450, "epoch": 13, "train_loss": 0.0013584281550720334}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9816, "test_loss": 0.06301712880134583}, {"step": 5500, "epoch": 14, "train_loss": 0.0008245377684943378}, {"step": 5550, "epoch": 14, "train_loss": 0.0006592019344680011}, {"step": 5600, "epoch": 14, "train_loss": 0.0014660698361694813}, {"step": 5650, "epoch": 14, "train_loss": 0.0009132636478170753}, {"step": 5700, "epoch": 14, "train_loss": 0.0005627474747598171}, {"step": 5750, "epoch": 14, "train_loss": 0.0013938703341409564}, {"step": 5800, "epoch": 14, "train_loss": 0.001884644152596593}, {"step": 5850, "epoch": 14, "train_loss": 0.001316796988248825}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9819, "test_loss": 0.0627727458000183}, {"step": 5900, "epoch": 15, "train_loss": 0.0011303852079436183}, {"step": 5950, "epoch": 15, "train_loss": 0.00158637051936239}, {"step": 6000, "epoch": 15, "train_loss": 0.0023287383373826742}, {"step": 6050, "epoch": 15, "train_loss": 0.0017517769010737538}, {"step": 6100, "epoch": 15, "train_loss": 0.0015438133850693703}, {"step": 6150, "epoch": 15, "train_loss": 0.000965150713454932}, {"step": 6200, "epoch": 15, "train_loss": 0.0009283519466407597}, {"step": 6250, "epoch": 15, "train_loss": 0.0008424912230111659}, {"epoch_end": 15, "step": 6256, "test_acc": 0.982, "test_loss": 0.06287671937942504}, {"step": 6300, "epoch": 16, "train_loss": 0.0011962049175053835}, {"step": 6350, "epoch": 16, "train_loss": 0.001000261981971562}, {"step": 6400, "epoch": 16, "train_loss": 0.0008840787340886891}, {"step": 6450, "epoch": 16, "train_loss": 0.00045400444651022553}, {"step": 6500, "epoch": 16, "train_loss": 0.001049683429300785}, {"step": 6550, "epoch": 16, "train_loss": 0.0011781944194808602}, {"step": 6600, "epoch": 16, "train_loss": 0.0008517744718119502}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9819, "test_loss": 0.06354984674453736}, {"step": 6650, "epoch": 17, "train_loss": 0.0005839620716869831}, {"step": 6700, "epoch": 17, "train_loss": 0.0003066162462346256}, {"step": 6750, "epoch": 17, "train_loss": 0.001399665605276823}, {"step": 6800, "epoch": 17, "train_loss": 0.0015314811607822776}, {"step": 6850, "epoch": 17, "train_loss": 0.0008560688002035022}, {"step": 6900, "epoch": 17, "train_loss": 0.001377783133648336}, {"step": 6950, "epoch": 17, "train_loss": 0.001105726812966168}, {"step": 7000, "epoch": 17, "train_loss": 0.0009439709829166532}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9819, "test_loss": 0.06338098063468933}, {"step": 7050, "epoch": 18, "train_loss": 0.0005534448428079486}, {"step": 7100, "epoch": 18, "train_loss": 0.0016684449510648847}, {"step": 7150, "epoch": 18, "train_loss": 0.00048497659736312926}, {"step": 7200, "epoch": 18, "train_loss": 0.0007778422441333532}, {"step": 7250, "epoch": 18, "train_loss": 0.001229303190484643}, {"step": 7300, "epoch": 18, "train_loss": 0.0006217429763637483}, {"step": 7350, "epoch": 18, "train_loss": 0.0007758463034406304}, {"step": 7400, "epoch": 18, "train_loss": 0.0005864178529009223}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9819, "test_loss": 0.0637749804019928}, {"step": 7450, "epoch": 19, "train_loss": 0.0005001954268664122}, {"step": 7500, "epoch": 19, "train_loss": 0.0009535456774756312}, {"step": 7550, "epoch": 19, "train_loss": 0.000578270002733916}, {"step": 7600, "epoch": 19, "train_loss": 0.0005295882001519203}, {"step": 7650, "epoch": 19, "train_loss": 0.0012070006923750043}, {"step": 7700, "epoch": 19, "train_loss": 0.0011975292582064867}, {"step": 7750, "epoch": 19, "train_loss": 0.0009928932413458824}, {"step": 7800, "epoch": 19, "train_loss": 0.0013849479146301746}, {"epoch_end": 19, "step": 7820, "test_acc": 0.982, "test_loss": 0.06412298550605774}, {"step": 7850, "epoch": 20, "train_loss": 0.0009324754355475307}, {"step": 7900, "epoch": 20, "train_loss": 0.0008891507750377059}, {"step": 7950, "epoch": 20, "train_loss": 0.0006962128099985421}, {"step": 8000, "epoch": 20, "train_loss": 0.0005678434972651303}, {"step": 8050, "epoch": 20, "train_loss": 0.0009835740784183145}, {"step": 8100, "epoch": 20, "train_loss": 0.000992418616078794}, {"step": 8150, "epoch": 20, "train_loss": 0.0008604043978266418}, {"step": 8200, "epoch": 20, "train_loss": 0.0005605288897641003}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9818, "test_loss": 0.0640114070892334}, {"step": 8250, "epoch": 21, "train_loss": 0.000887602916918695}, {"step": 8300, "epoch": 21, "train_loss": 0.000894980039447546}, {"step": 8350, "epoch": 21, "train_loss": 0.0009103642660193145}, {"step": 8400, "epoch": 21, "train_loss": 0.0006621874636039138}, {"step": 8450, "epoch": 21, "train_loss": 0.0008148560300469398}, {"step": 8500, "epoch": 21, "train_loss": 0.000907673966139555}, {"step": 8550, "epoch": 21, "train_loss": 0.0006180392811074853}, {"step": 8600, "epoch": 21, "train_loss": 0.0010829214006662369}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9818, "test_loss": 0.06463181204795837}, {"step": 8650, "epoch": 22, "train_loss": 0.0004346754285506904}, {"step": 8700, "epoch": 22, "train_loss": 0.0005230087554082274}, {"step": 8750, "epoch": 22, "train_loss": 0.0005779966595582664}, {"step": 8800, "epoch": 22, "train_loss": 0.0009458012646064162}, {"step": 8850, "epoch": 22, "train_loss": 0.0007219917606562376}, {"step": 8900, "epoch": 22, "train_loss": 0.0009516506688669324}, {"step": 8950, "epoch": 22, "train_loss": 0.0006183352670632303}, {"epoch_end": 22, "step": 8993, "test_acc": 0.982, "test_loss": 0.06442908101081848}, {"step": 9000, "epoch": 23, "train_loss": 0.0005609735380858183}, {"step": 9050, "epoch": 23, "train_loss": 0.0007887763786129653}, {"step": 9100, "epoch": 23, "train_loss": 0.0006966495420783758}, {"step": 9150, "epoch": 23, "train_loss": 0.0010859654285013676}, {"step": 9200, "epoch": 23, "train_loss": 0.0009497366845607758}, {"step": 9250, "epoch": 23, "train_loss": 0.0007913994486443698}, {"step": 9300, "epoch": 23, "train_loss": 0.0006284067058004439}, {"step": 9350, "epoch": 23, "train_loss": 0.0006204663077369332}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9822, "test_loss": 0.06480937170982361}, {"step": 9400, "epoch": 24, "train_loss": 0.0006771076004952192}, {"step": 9450, "epoch": 24, "train_loss": 0.0004173183988314122}, {"step": 9500, "epoch": 24, "train_loss": 0.0008216394344344735}, {"step": 9550, "epoch": 24, "train_loss": 0.0006163725629448891}, {"step": 9600, "epoch": 24, "train_loss": 0.0006746436702087522}, {"step": 9650, "epoch": 24, "train_loss": 0.0010440711630508304}, {"step": 9700, "epoch": 24, "train_loss": 0.0005398837383836508}, {"step": 9750, "epoch": 24, "train_loss": 0.0013254232471808791}, {"epoch_end": 24, "step": 9775, "test_acc": 0.982, "test_loss": 0.06484736838340759}], "final": {"test_acc": 0.982, "test_loss": 0.06484736838340759, "wall_s": 14.403992414474487}} \ No newline at end of file diff --git a/results/ep_archmatch_v1_mnist_dfa_w500_d1_s0.json b/results/ep_archmatch_v1_mnist_dfa_w500_d1_s0.json new file mode 100644 index 0000000..dc32418 --- /dev/null +++ b/results/ep_archmatch_v1_mnist_dfa_w500_d1_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "mnist", "depth": 1, "width": 500, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_archmatch_v1_mnist_dfa_w500_d1_s0", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.529653549194336, "cos_r_negg": [0.007927153259515762], "cos_innovation_negg": [0.007927153259515762], "cos_apical_negg": [0.007927153259515762], "cos_Ac_negg": [0.007927153259515762], "r_norm": [6.919894218444824]}, {"step": 50, "epoch": 0, "train_loss": 0.5378985404968262, "cos_r_negg": [0.446338027715683], "cos_innovation_negg": [0.446338027715683], "cos_apical_negg": [0.446338027715683], "cos_Ac_negg": [0.446338027715683], "r_norm": [1.4515876770019531]}, {"step": 100, "epoch": 0, "train_loss": 0.15611110627651215, "cos_r_negg": [0.4595390260219574], "cos_innovation_negg": [0.4595390260219574], "cos_apical_negg": [0.4595390260219574], "cos_Ac_negg": [0.4595390260219574], "r_norm": [1.1902644634246826]}, {"step": 150, "epoch": 0, "train_loss": 0.17079968750476837, "cos_r_negg": [0.46849602460861206], "cos_innovation_negg": [0.46849602460861206], "cos_apical_negg": [0.46849602460861206], "cos_Ac_negg": [0.46849602460861206], "r_norm": [1.005897045135498]}, {"step": 200, "epoch": 0, "train_loss": 0.3053852319717407, "cos_r_negg": [0.47044897079467773], "cos_innovation_negg": [0.47044897079467773], "cos_apical_negg": [0.47044897079467773], "cos_Ac_negg": [0.47044897079467773], "r_norm": [1.0187816619873047]}, {"step": 250, "epoch": 0, "train_loss": 0.2931833863258362, "cos_r_negg": [0.4832371175289154], "cos_innovation_negg": [0.4832371175289154], "cos_apical_negg": [0.4832371175289154], "cos_Ac_negg": [0.4832371175289154], "r_norm": [0.8806146383285522]}, {"step": 300, "epoch": 0, "train_loss": 0.1320013403892517, "cos_r_negg": [0.4842876195907593], "cos_innovation_negg": [0.4842876195907593], "cos_apical_negg": [0.4842876195907593], "cos_Ac_negg": [0.4842876195907593], "r_norm": [0.788491427898407]}, {"step": 350, "epoch": 0, "train_loss": 0.23352736234664917, "cos_r_negg": [0.5069469809532166], "cos_innovation_negg": [0.5069469809532166], "cos_apical_negg": [0.5069469809532166], "cos_Ac_negg": [0.5069469809532166], "r_norm": [0.6924450397491455]}, {"epoch_end": 0, "step": 391, "test_acc": 0.9526, "test_loss": 0.16421128311157226}, {"step": 400, "epoch": 1, "train_loss": 0.12783637642860413, "cos_r_negg": [0.5114192962646484], "cos_innovation_negg": [0.5114192962646484], "cos_apical_negg": [0.5114192962646484], "cos_Ac_negg": [0.5114192962646484], "r_norm": [0.6613460779190063]}, {"step": 450, "epoch": 1, "train_loss": 0.13752709329128265, "cos_r_negg": [0.5126492381095886], "cos_innovation_negg": [0.5126492381095886], "cos_apical_negg": [0.5126492381095886], "cos_Ac_negg": [0.5126492381095886], "r_norm": [0.5940003395080566]}, {"step": 500, "epoch": 1, "train_loss": 0.207707479596138, "cos_r_negg": [0.5260812640190125], "cos_innovation_negg": [0.5260812640190125], "cos_apical_negg": [0.5260812640190125], "cos_Ac_negg": [0.5260812640190125], "r_norm": [0.41802847385406494]}, {"step": 550, "epoch": 1, "train_loss": 0.09028483927249908, "cos_r_negg": [0.5275427103042603], "cos_innovation_negg": [0.5275427103042603], "cos_apical_negg": [0.5275427103042603], "cos_Ac_negg": [0.5275427103042603], "r_norm": [0.5970697999000549]}, {"step": 600, "epoch": 1, "train_loss": 0.056863557547330856, "cos_r_negg": [0.5313643217086792], "cos_innovation_negg": [0.5313643217086792], "cos_apical_negg": [0.5313643217086792], "cos_Ac_negg": [0.5313643217086792], "r_norm": [0.4569489061832428]}, {"step": 650, "epoch": 1, "train_loss": 0.05803891643881798, "cos_r_negg": [0.540709376335144], "cos_innovation_negg": [0.540709376335144], "cos_apical_negg": [0.540709376335144], "cos_Ac_negg": [0.540709376335144], "r_norm": [0.4741382896900177]}, {"step": 700, "epoch": 1, "train_loss": 0.08346912264823914, "cos_r_negg": [0.5343747138977051], "cos_innovation_negg": [0.5343747138977051], "cos_apical_negg": [0.5343747138977051], "cos_Ac_negg": [0.5343747138977051], "r_norm": [0.5042672157287598]}, {"step": 750, "epoch": 1, "train_loss": 0.15645092725753784, "cos_r_negg": [0.5503103733062744], "cos_innovation_negg": [0.5503103733062744], "cos_apical_negg": [0.5503103733062744], "cos_Ac_negg": [0.5503103733062744], "r_norm": [0.6418039798736572]}, {"epoch_end": 1, "step": 782, "test_acc": 0.9588, "test_loss": 0.13556932945251465}, {"step": 800, "epoch": 2, "train_loss": 0.09077601879835129, "cos_r_negg": [0.5468267798423767], "cos_innovation_negg": [0.5468267798423767], "cos_apical_negg": [0.5468267798423767], "cos_Ac_negg": [0.5468267798423767], "r_norm": [0.45994025468826294]}, {"step": 850, "epoch": 2, "train_loss": 0.04424097761511803, "cos_r_negg": [0.5566318035125732], "cos_innovation_negg": [0.5566318035125732], "cos_apical_negg": [0.5566318035125732], "cos_Ac_negg": [0.5566318035125732], "r_norm": [0.42781031131744385]}, {"step": 900, "epoch": 2, "train_loss": 0.07376974076032639, "cos_r_negg": [0.5656780004501343], "cos_innovation_negg": [0.5656780004501343], "cos_apical_negg": [0.5656780004501343], "cos_Ac_negg": [0.5656780004501343], "r_norm": [0.4573560655117035]}, {"step": 950, "epoch": 2, "train_loss": 0.08151404559612274, "cos_r_negg": [0.5647845268249512], "cos_innovation_negg": [0.5647845268249512], "cos_apical_negg": [0.5647845268249512], "cos_Ac_negg": [0.5647845268249512], "r_norm": [0.36703014373779297]}, {"step": 1000, "epoch": 2, "train_loss": 0.06810769438743591, "cos_r_negg": [0.5567589998245239], "cos_innovation_negg": [0.5567589998245239], "cos_apical_negg": [0.5567589998245239], "cos_Ac_negg": [0.5567589998245239], "r_norm": [0.38407331705093384]}, {"step": 1050, "epoch": 2, "train_loss": 0.03570346534252167, "cos_r_negg": [0.5609099864959717], "cos_innovation_negg": [0.5609099864959717], "cos_apical_negg": [0.5609099864959717], "cos_Ac_negg": [0.5609099864959717], "r_norm": [0.4029432535171509]}, {"step": 1100, "epoch": 2, "train_loss": 0.13104398548603058, "cos_r_negg": [0.5644813179969788], "cos_innovation_negg": [0.5644813179969788], "cos_apical_negg": [0.5644813179969788], "cos_Ac_negg": [0.5644813179969788], "r_norm": [0.3724943697452545]}, {"step": 1150, "epoch": 2, "train_loss": 0.0787217915058136, "cos_r_negg": [0.5776868462562561], "cos_innovation_negg": [0.5776868462562561], "cos_apical_negg": [0.5776868462562561], "cos_Ac_negg": [0.5776868462562561], "r_norm": [0.3190084993839264]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9687, "test_loss": 0.10461063003540039}, {"step": 1200, "epoch": 3, "train_loss": 0.03989540785551071, "cos_r_negg": [0.5671805739402771], "cos_innovation_negg": [0.5671805739402771], "cos_apical_negg": [0.5671805739402771], "cos_Ac_negg": [0.5671805739402771], "r_norm": [0.3714447617530823]}, {"step": 1250, "epoch": 3, "train_loss": 0.07401026785373688, "cos_r_negg": [0.5758885145187378], "cos_innovation_negg": [0.5758885145187378], "cos_apical_negg": [0.5758885145187378], "cos_Ac_negg": [0.5758885145187378], "r_norm": [0.34470659494400024]}, {"step": 1300, "epoch": 3, "train_loss": 0.0295491311699152, "cos_r_negg": [0.5792849063873291], "cos_innovation_negg": [0.5792849063873291], "cos_apical_negg": [0.5792849063873291], "cos_Ac_negg": [0.5792849063873291], "r_norm": [0.33078885078430176]}, {"step": 1350, "epoch": 3, "train_loss": 0.013916345313191414, "cos_r_negg": [0.5819554328918457], "cos_innovation_negg": [0.5819554328918457], "cos_apical_negg": [0.5819554328918457], "cos_Ac_negg": [0.5819554328918457], "r_norm": [0.30006662011146545]}, {"step": 1400, "epoch": 3, "train_loss": 0.037473201751708984, "cos_r_negg": [0.5953552722930908], "cos_innovation_negg": [0.5953552722930908], "cos_apical_negg": [0.5953552722930908], "cos_Ac_negg": [0.5953552722930908], "r_norm": [0.34922927618026733]}, {"step": 1450, "epoch": 3, "train_loss": 0.018525030463933945, "cos_r_negg": [0.5993934273719788], "cos_innovation_negg": [0.5993934273719788], "cos_apical_negg": [0.5993934273719788], "cos_Ac_negg": [0.5993934273719788], "r_norm": [0.344095379114151]}, {"step": 1500, "epoch": 3, "train_loss": 0.07173670828342438, "cos_r_negg": [0.5991964340209961], "cos_innovation_negg": [0.5991964340209961], "cos_apical_negg": [0.5991964340209961], "cos_Ac_negg": [0.5991964340209961], "r_norm": [0.24706979095935822]}, {"step": 1550, "epoch": 3, "train_loss": 0.03505013510584831, "cos_r_negg": [0.5985463261604309], "cos_innovation_negg": [0.5985463261604309], "cos_apical_negg": [0.5985463261604309], "cos_Ac_negg": [0.5985463261604309], "r_norm": [0.28853869438171387]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9668, "test_loss": 0.11504416694641113}, {"step": 1600, "epoch": 4, "train_loss": 0.06070321053266525, "cos_r_negg": [0.6009326577186584], "cos_innovation_negg": [0.6009326577186584], "cos_apical_negg": [0.6009326577186584], "cos_Ac_negg": [0.6009326577186584], "r_norm": [0.3139006793498993]}, {"step": 1650, "epoch": 4, "train_loss": 0.0654536783695221, "cos_r_negg": [0.6011091470718384], "cos_innovation_negg": [0.6011091470718384], "cos_apical_negg": [0.6011091470718384], "cos_Ac_negg": [0.6011091470718384], "r_norm": [0.2972111999988556]}, {"step": 1700, "epoch": 4, "train_loss": 0.058433350175619125, "cos_r_negg": [0.6047418117523193], "cos_innovation_negg": [0.6047418117523193], "cos_apical_negg": [0.6047418117523193], "cos_Ac_negg": [0.6047418117523193], "r_norm": [0.2860991656780243]}, {"step": 1750, "epoch": 4, "train_loss": 0.04591287672519684, "cos_r_negg": [0.6114698648452759], "cos_innovation_negg": [0.6114698648452759], "cos_apical_negg": [0.6114698648452759], "cos_Ac_negg": [0.6114698648452759], "r_norm": [0.2644854485988617]}, {"step": 1800, "epoch": 4, "train_loss": 0.019722694531083107, "cos_r_negg": [0.6019526720046997], "cos_innovation_negg": [0.6019526720046997], "cos_apical_negg": [0.6019526720046997], "cos_Ac_negg": [0.6019526720046997], "r_norm": [0.24991106986999512]}, {"step": 1850, "epoch": 4, "train_loss": 0.07621756196022034, "cos_r_negg": [0.6084581017494202], "cos_innovation_negg": [0.6084581017494202], "cos_apical_negg": [0.6084581017494202], "cos_Ac_negg": [0.6084581017494202], "r_norm": [0.2757892906665802]}, {"step": 1900, "epoch": 4, "train_loss": 0.00765196792781353, "cos_r_negg": [0.6242289543151855], "cos_innovation_negg": [0.6242289543151855], "cos_apical_negg": [0.6242289543151855], "cos_Ac_negg": [0.6242289543151855], "r_norm": [0.22341588139533997]}, {"step": 1950, "epoch": 4, "train_loss": 0.07084986567497253, "cos_r_negg": [0.6056423187255859], "cos_innovation_negg": [0.6056423187255859], "cos_apical_negg": [0.6056423187255859], "cos_Ac_negg": [0.6056423187255859], "r_norm": [0.2361248880624771]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9723, "test_loss": 0.09193439445495606}, {"step": 2000, "epoch": 5, "train_loss": 0.022296397015452385, "cos_r_negg": [0.6212233304977417], "cos_innovation_negg": [0.6212233304977417], "cos_apical_negg": [0.6212233304977417], "cos_Ac_negg": [0.6212233304977417], "r_norm": [0.23196306824684143]}, {"step": 2050, "epoch": 5, "train_loss": 0.013410639949142933, "cos_r_negg": [0.6127545237541199], "cos_innovation_negg": [0.6127545237541199], "cos_apical_negg": [0.6127545237541199], "cos_Ac_negg": [0.6127545237541199], "r_norm": [0.25504547357559204]}, {"step": 2100, "epoch": 5, "train_loss": 0.010225128382444382, "cos_r_negg": [0.6228626370429993], "cos_innovation_negg": [0.6228626370429993], "cos_apical_negg": [0.6228626370429993], "cos_Ac_negg": [0.6228626370429993], "r_norm": [0.23373067378997803]}, {"step": 2150, "epoch": 5, "train_loss": 0.0036907277535647154, "cos_r_negg": [0.6145119667053223], "cos_innovation_negg": [0.6145119667053223], "cos_apical_negg": [0.6145119667053223], "cos_Ac_negg": [0.6145119667053223], "r_norm": [0.2328505963087082]}, {"step": 2200, "epoch": 5, "train_loss": 0.01594649627804756, "cos_r_negg": [0.6221743822097778], "cos_innovation_negg": [0.6221743822097778], "cos_apical_negg": [0.6221743822097778], "cos_Ac_negg": [0.6221743822097778], "r_norm": [0.24157926440238953]}, {"step": 2250, "epoch": 5, "train_loss": 0.01261073350906372, "cos_r_negg": [0.6334763169288635], "cos_innovation_negg": [0.6334763169288635], "cos_apical_negg": [0.6334763169288635], "cos_Ac_negg": [0.6334763169288635], "r_norm": [0.2142297923564911]}, {"step": 2300, "epoch": 5, "train_loss": 0.011496011167764664, "cos_r_negg": [0.6314460039138794], "cos_innovation_negg": [0.6314460039138794], "cos_apical_negg": [0.6314460039138794], "cos_Ac_negg": [0.6314460039138794], "r_norm": [0.2083812803030014]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9736, "test_loss": 0.09129757633209229}, {"step": 2350, "epoch": 6, "train_loss": 0.016033321619033813, "cos_r_negg": [0.6286594867706299], "cos_innovation_negg": [0.6286594867706299], "cos_apical_negg": [0.6286594867706299], "cos_Ac_negg": [0.6286594867706299], "r_norm": [0.21533192694187164]}, {"step": 2400, "epoch": 6, "train_loss": 0.0075647542253136635, "cos_r_negg": [0.6392087340354919], "cos_innovation_negg": [0.6392087340354919], "cos_apical_negg": [0.6392087340354919], "cos_Ac_negg": [0.6392087340354919], "r_norm": [0.23080016672611237]}, {"step": 2450, "epoch": 6, "train_loss": 0.01589258573949337, "cos_r_negg": [0.6323435306549072], "cos_innovation_negg": [0.6323435306549072], "cos_apical_negg": [0.6323435306549072], "cos_Ac_negg": [0.6323435306549072], "r_norm": [0.23569393157958984]}, {"step": 2500, "epoch": 6, "train_loss": 0.013036523014307022, "cos_r_negg": [0.6303362250328064], "cos_innovation_negg": [0.6303362250328064], "cos_apical_negg": [0.6303362250328064], "cos_Ac_negg": [0.6303362250328064], "r_norm": [0.2163347452878952]}, {"step": 2550, "epoch": 6, "train_loss": 0.01871556043624878, "cos_r_negg": [0.6301670074462891], "cos_innovation_negg": [0.6301670074462891], "cos_apical_negg": [0.6301670074462891], "cos_Ac_negg": [0.6301670074462891], "r_norm": [0.18959662318229675]}, {"step": 2600, "epoch": 6, "train_loss": 0.007934422232210636, "cos_r_negg": [0.6363259553909302], "cos_innovation_negg": [0.6363259553909302], "cos_apical_negg": [0.6363259553909302], "cos_Ac_negg": [0.6363259553909302], "r_norm": [0.2006957232952118]}, {"step": 2650, "epoch": 6, "train_loss": 0.02632422000169754, "cos_r_negg": [0.6353095769882202], "cos_innovation_negg": [0.6353095769882202], "cos_apical_negg": [0.6353095769882202], "cos_Ac_negg": [0.6353095769882202], "r_norm": [0.21200628578662872]}, {"step": 2700, "epoch": 6, "train_loss": 0.0036291200667619705, "cos_r_negg": [0.6354855298995972], "cos_innovation_negg": [0.6354855298995972], "cos_apical_negg": [0.6354855298995972], "cos_Ac_negg": [0.6354855298995972], "r_norm": [0.24552160501480103]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9764, "test_loss": 0.08520364227294921}, {"step": 2750, "epoch": 7, "train_loss": 0.0025059834588319063, "cos_r_negg": [0.6370502710342407], "cos_innovation_negg": [0.6370502710342407], "cos_apical_negg": [0.6370502710342407], "cos_Ac_negg": [0.6370502710342407], "r_norm": [0.21230348944664001]}, {"step": 2800, "epoch": 7, "train_loss": 0.0032542741391807795, "cos_r_negg": [0.6403782367706299], "cos_innovation_negg": [0.6403782367706299], "cos_apical_negg": [0.6403782367706299], "cos_Ac_negg": [0.6403782367706299], "r_norm": [0.21699409186840057]}, {"step": 2850, "epoch": 7, "train_loss": 0.003031367203220725, "cos_r_negg": [0.6405731439590454], "cos_innovation_negg": [0.6405731439590454], "cos_apical_negg": [0.6405731439590454], "cos_Ac_negg": [0.6405731439590454], "r_norm": [0.19857990741729736]}, {"step": 2900, "epoch": 7, "train_loss": 0.004416514188051224, "cos_r_negg": [0.6357875466346741], "cos_innovation_negg": [0.6357875466346741], "cos_apical_negg": [0.6357875466346741], "cos_Ac_negg": [0.6357875466346741], "r_norm": [0.1913098394870758]}, {"step": 2950, "epoch": 7, "train_loss": 0.0033339718356728554, "cos_r_negg": [0.6328017115592957], "cos_innovation_negg": [0.6328017115592957], "cos_apical_negg": [0.6328017115592957], "cos_Ac_negg": [0.6328017115592957], "r_norm": [0.19515562057495117]}, {"step": 3000, "epoch": 7, "train_loss": 0.008366161957383156, "cos_r_negg": [0.6361695528030396], "cos_innovation_negg": [0.6361695528030396], "cos_apical_negg": [0.6361695528030396], "cos_Ac_negg": [0.6361695528030396], "r_norm": [0.21515579521656036]}, {"step": 3050, "epoch": 7, "train_loss": 0.004386312328279018, "cos_r_negg": [0.6459784507751465], "cos_innovation_negg": [0.6459784507751465], "cos_apical_negg": [0.6459784507751465], "cos_Ac_negg": [0.6459784507751465], "r_norm": [0.21083509922027588]}, {"step": 3100, "epoch": 7, "train_loss": 0.0035264105536043644, "cos_r_negg": [0.6453342437744141], "cos_innovation_negg": [0.6453342437744141], "cos_apical_negg": [0.6453342437744141], "cos_Ac_negg": [0.6453342437744141], "r_norm": [0.1780584156513214]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9753, "test_loss": 0.08862885894775391}, {"step": 3150, "epoch": 8, "train_loss": 0.003042601514607668, "cos_r_negg": [0.6433063745498657], "cos_innovation_negg": [0.6433063745498657], "cos_apical_negg": [0.6433063745498657], "cos_Ac_negg": [0.6433063745498657], "r_norm": [0.19454246759414673]}, {"step": 3200, "epoch": 8, "train_loss": 0.011032410897314548, "cos_r_negg": [0.644295871257782], "cos_innovation_negg": [0.644295871257782], "cos_apical_negg": [0.644295871257782], "cos_Ac_negg": [0.644295871257782], "r_norm": [0.20710989832878113]}, {"step": 3250, "epoch": 8, "train_loss": 0.0029919464141130447, "cos_r_negg": [0.6445097923278809], "cos_innovation_negg": [0.6445097923278809], "cos_apical_negg": [0.6445097923278809], "cos_Ac_negg": [0.6445097923278809], "r_norm": [0.20775106549263]}, {"step": 3300, "epoch": 8, "train_loss": 0.0023679169826209545, "cos_r_negg": [0.644466757774353], "cos_innovation_negg": [0.644466757774353], "cos_apical_negg": [0.644466757774353], "cos_Ac_negg": [0.644466757774353], "r_norm": [0.191869854927063]}, {"step": 3350, "epoch": 8, "train_loss": 0.006431836634874344, "cos_r_negg": [0.648067057132721], "cos_innovation_negg": [0.648067057132721], "cos_apical_negg": [0.648067057132721], "cos_Ac_negg": [0.648067057132721], "r_norm": [0.19135840237140656]}, {"step": 3400, "epoch": 8, "train_loss": 0.00398094579577446, "cos_r_negg": [0.6451742649078369], "cos_innovation_negg": [0.6451742649078369], "cos_apical_negg": [0.6451742649078369], "cos_Ac_negg": [0.6451742649078369], "r_norm": [0.20446622371673584]}, {"step": 3450, "epoch": 8, "train_loss": 0.014922883361577988, "cos_r_negg": [0.6469951868057251], "cos_innovation_negg": [0.6469951868057251], "cos_apical_negg": [0.6469951868057251], "cos_Ac_negg": [0.6469951868057251], "r_norm": [0.2097967118024826]}, {"step": 3500, "epoch": 8, "train_loss": 0.0067122867330908775, "cos_r_negg": [0.6454347372055054], "cos_innovation_negg": [0.6454347372055054], "cos_apical_negg": [0.6454347372055054], "cos_Ac_negg": [0.6454347372055054], "r_norm": [0.19009284675121307]}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9755, "test_loss": 0.08785459270477294}, {"step": 3550, "epoch": 9, "train_loss": 0.003050497267395258, "cos_r_negg": [0.6495440602302551], "cos_innovation_negg": [0.6495440602302551], "cos_apical_negg": [0.6495440602302551], "cos_Ac_negg": [0.6495440602302551], "r_norm": [0.20186209678649902]}, {"step": 3600, "epoch": 9, "train_loss": 0.003470212686806917, "cos_r_negg": [0.650248646736145], "cos_innovation_negg": [0.650248646736145], "cos_apical_negg": [0.650248646736145], "cos_Ac_negg": [0.650248646736145], "r_norm": [0.18642909824848175]}, {"step": 3650, "epoch": 9, "train_loss": 0.005127946846187115, "cos_r_negg": [0.6513925194740295], "cos_innovation_negg": [0.6513925194740295], "cos_apical_negg": [0.6513925194740295], "cos_Ac_negg": [0.6513925194740295], "r_norm": [0.17918813228607178]}, {"step": 3700, "epoch": 9, "train_loss": 0.0054059759713709354, "cos_r_negg": [0.6479052305221558], "cos_innovation_negg": [0.6479052305221558], "cos_apical_negg": [0.6479052305221558], "cos_Ac_negg": [0.6479052305221558], "r_norm": [0.1881476640701294]}, {"step": 3750, "epoch": 9, "train_loss": 0.005067600402981043, "cos_r_negg": [0.6526027321815491], "cos_innovation_negg": [0.6526027321815491], "cos_apical_negg": [0.6526027321815491], "cos_Ac_negg": [0.6526027321815491], "r_norm": [0.1800059676170349]}, {"step": 3800, "epoch": 9, "train_loss": 0.002939029596745968, "cos_r_negg": [0.6529317498207092], "cos_innovation_negg": [0.6529317498207092], "cos_apical_negg": [0.6529317498207092], "cos_Ac_negg": [0.6529317498207092], "r_norm": [0.18275636434555054]}, {"step": 3850, "epoch": 9, "train_loss": 0.00386594096198678, "cos_r_negg": [0.6545953750610352], "cos_innovation_negg": [0.6545953750610352], "cos_apical_negg": [0.6545953750610352], "cos_Ac_negg": [0.6545953750610352], "r_norm": [0.19248118996620178]}, {"step": 3900, "epoch": 9, "train_loss": 0.004301756154745817, "cos_r_negg": [0.649564266204834], "cos_innovation_negg": [0.649564266204834], "cos_apical_negg": [0.649564266204834], "cos_Ac_negg": [0.649564266204834], "r_norm": [0.17353592813014984]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9755, "test_loss": 0.08616431102752685}, {"step": 3950, "epoch": 10, "train_loss": 0.002714196452870965, "cos_r_negg": [0.6493533849716187], "cos_innovation_negg": [0.6493533849716187], "cos_apical_negg": [0.6493533849716187], "cos_Ac_negg": [0.6493533849716187], "r_norm": [0.19229425489902496]}, {"step": 4000, "epoch": 10, "train_loss": 0.003034464782103896, "cos_r_negg": [0.6500505208969116], "cos_innovation_negg": [0.6500505208969116], "cos_apical_negg": [0.6500505208969116], "cos_Ac_negg": [0.6500505208969116], "r_norm": [0.19248169660568237]}, {"step": 4050, "epoch": 10, "train_loss": 0.0018771615577861667, "cos_r_negg": [0.6483029127120972], "cos_innovation_negg": [0.6483029127120972], "cos_apical_negg": [0.6483029127120972], "cos_Ac_negg": [0.6483029127120972], "r_norm": [0.18626996874809265]}, {"step": 4100, "epoch": 10, "train_loss": 0.0024207872338593006, "cos_r_negg": [0.6539337038993835], "cos_innovation_negg": [0.6539337038993835], "cos_apical_negg": [0.6539337038993835], "cos_Ac_negg": [0.6539337038993835], "r_norm": [0.18321529030799866]}, {"step": 4150, "epoch": 10, "train_loss": 0.001863198122009635, "cos_r_negg": [0.6543848514556885], "cos_innovation_negg": [0.6543848514556885], "cos_apical_negg": [0.6543848514556885], "cos_Ac_negg": [0.6543848514556885], "r_norm": [0.16975359618663788]}, {"step": 4200, "epoch": 10, "train_loss": 0.004063147585839033, "cos_r_negg": [0.655256450176239], "cos_innovation_negg": [0.655256450176239], "cos_apical_negg": [0.655256450176239], "cos_Ac_negg": [0.655256450176239], "r_norm": [0.17035937309265137]}, {"step": 4250, "epoch": 10, "train_loss": 0.003506137989461422, "cos_r_negg": [0.6565347909927368], "cos_innovation_negg": [0.6565347909927368], "cos_apical_negg": [0.6565347909927368], "cos_Ac_negg": [0.6565347909927368], "r_norm": [0.18507790565490723]}, {"step": 4300, "epoch": 10, "train_loss": 0.0017705544596537948, "cos_r_negg": [0.6559028625488281], "cos_innovation_negg": [0.6559028625488281], "cos_apical_negg": [0.6559028625488281], "cos_Ac_negg": [0.6559028625488281], "r_norm": [0.17871400713920593]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9765, "test_loss": 0.08781707572937011}, {"step": 4350, "epoch": 11, "train_loss": 0.004004799760878086, "cos_r_negg": [0.6556627750396729], "cos_innovation_negg": [0.6556627750396729], "cos_apical_negg": [0.6556627750396729], "cos_Ac_negg": [0.6556627750396729], "r_norm": [0.17836390435695648]}, {"step": 4400, "epoch": 11, "train_loss": 0.001953459344804287, "cos_r_negg": [0.6563931703567505], "cos_innovation_negg": [0.6563931703567505], "cos_apical_negg": [0.6563931703567505], "cos_Ac_negg": [0.6563931703567505], "r_norm": [0.1793738603591919]}, {"step": 4450, "epoch": 11, "train_loss": 0.0035812892019748688, "cos_r_negg": [0.656152069568634], "cos_innovation_negg": [0.656152069568634], "cos_apical_negg": [0.656152069568634], "cos_Ac_negg": [0.656152069568634], "r_norm": [0.17429597675800323]}, {"step": 4500, "epoch": 11, "train_loss": 0.001329883118160069, "cos_r_negg": [0.658604621887207], "cos_innovation_negg": [0.658604621887207], "cos_apical_negg": [0.658604621887207], "cos_Ac_negg": [0.658604621887207], "r_norm": [0.18984994292259216]}, {"step": 4550, "epoch": 11, "train_loss": 0.004240573383867741, "cos_r_negg": [0.656351625919342], "cos_innovation_negg": [0.656351625919342], "cos_apical_negg": [0.656351625919342], "cos_Ac_negg": [0.656351625919342], "r_norm": [0.19123215973377228]}, {"step": 4600, "epoch": 11, "train_loss": 0.0010742396116256714, "cos_r_negg": [0.656906247138977], "cos_innovation_negg": [0.656906247138977], "cos_apical_negg": [0.656906247138977], "cos_Ac_negg": [0.656906247138977], "r_norm": [0.17487865686416626]}, {"step": 4650, "epoch": 11, "train_loss": 0.002154822926968336, "cos_r_negg": [0.6589900255203247], "cos_innovation_negg": [0.6589900255203247], "cos_apical_negg": [0.6589900255203247], "cos_Ac_negg": [0.6589900255203247], "r_norm": [0.18067440390586853]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9758, "test_loss": 0.0886783088684082}, {"step": 4700, "epoch": 12, "train_loss": 0.0032127376180142164, "cos_r_negg": [0.6574692726135254], "cos_innovation_negg": [0.6574692726135254], "cos_apical_negg": [0.6574692726135254], "cos_Ac_negg": [0.6574692726135254], "r_norm": [0.17950871586799622]}, {"step": 4750, "epoch": 12, "train_loss": 0.002027628244832158, "cos_r_negg": [0.659750759601593], "cos_innovation_negg": [0.659750759601593], "cos_apical_negg": [0.659750759601593], "cos_Ac_negg": [0.659750759601593], "r_norm": [0.18223021924495697]}, {"step": 4800, "epoch": 12, "train_loss": 0.002229973440989852, "cos_r_negg": [0.6575031876564026], "cos_innovation_negg": [0.6575031876564026], "cos_apical_negg": [0.6575031876564026], "cos_Ac_negg": [0.6575031876564026], "r_norm": [0.18359892070293427]}, {"step": 4850, "epoch": 12, "train_loss": 0.001674095168709755, "cos_r_negg": [0.6567223072052002], "cos_innovation_negg": [0.6567223072052002], "cos_apical_negg": [0.6567223072052002], "cos_Ac_negg": [0.6567223072052002], "r_norm": [0.18486355245113373]}, {"step": 4900, "epoch": 12, "train_loss": 0.001528049586340785, "cos_r_negg": [0.6567079424858093], "cos_innovation_negg": [0.6567079424858093], "cos_apical_negg": [0.6567079424858093], "cos_Ac_negg": [0.6567079424858093], "r_norm": [0.18663716316223145]}, {"step": 4950, "epoch": 12, "train_loss": 0.00165956886485219, "cos_r_negg": [0.6557912826538086], "cos_innovation_negg": [0.6557912826538086], "cos_apical_negg": [0.6557912826538086], "cos_Ac_negg": [0.6557912826538086], "r_norm": [0.17783990502357483]}, {"step": 5000, "epoch": 12, "train_loss": 0.0017242147587239742, "cos_r_negg": [0.6599199771881104], "cos_innovation_negg": [0.6599199771881104], "cos_apical_negg": [0.6599199771881104], "cos_Ac_negg": [0.6599199771881104], "r_norm": [0.17558735609054565]}, {"step": 5050, "epoch": 12, "train_loss": 0.0022029599640518427, "cos_r_negg": [0.659331202507019], "cos_innovation_negg": [0.659331202507019], "cos_apical_negg": [0.659331202507019], "cos_Ac_negg": [0.659331202507019], "r_norm": [0.17106519639492035]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9761, "test_loss": 0.08887008666992187}, {"step": 5100, "epoch": 13, "train_loss": 0.001803007093258202, "cos_r_negg": [0.6594165563583374], "cos_innovation_negg": [0.6594165563583374], "cos_apical_negg": [0.6594165563583374], "cos_Ac_negg": [0.6594165563583374], "r_norm": [0.17785564064979553]}, {"step": 5150, "epoch": 13, "train_loss": 0.004073751624673605, "cos_r_negg": [0.661300539970398], "cos_innovation_negg": [0.661300539970398], "cos_apical_negg": [0.661300539970398], "cos_Ac_negg": [0.661300539970398], "r_norm": [0.1819925457239151]}, {"step": 5200, "epoch": 13, "train_loss": 0.0028512233402580023, "cos_r_negg": [0.6621579527854919], "cos_innovation_negg": [0.6621579527854919], "cos_apical_negg": [0.6621579527854919], "cos_Ac_negg": [0.6621579527854919], "r_norm": [0.1874617636203766]}, {"step": 5250, "epoch": 13, "train_loss": 0.0018842148128896952, "cos_r_negg": [0.6602969169616699], "cos_innovation_negg": [0.6602969169616699], "cos_apical_negg": [0.6602969169616699], "cos_Ac_negg": [0.6602969169616699], "r_norm": [0.18164794147014618]}, {"step": 5300, "epoch": 13, "train_loss": 0.001935745240189135, "cos_r_negg": [0.6624356508255005], "cos_innovation_negg": [0.6624356508255005], "cos_apical_negg": [0.6624356508255005], "cos_Ac_negg": [0.6624356508255005], "r_norm": [0.17963793873786926]}, {"step": 5350, "epoch": 13, "train_loss": 0.002686596941202879, "cos_r_negg": [0.6628423929214478], "cos_innovation_negg": [0.6628423929214478], "cos_apical_negg": [0.6628423929214478], "cos_Ac_negg": [0.6628423929214478], "r_norm": [0.1857079118490219]}, {"step": 5400, "epoch": 13, "train_loss": 0.0015342553379014134, "cos_r_negg": [0.6614811420440674], "cos_innovation_negg": [0.6614811420440674], "cos_apical_negg": [0.6614811420440674], "cos_Ac_negg": [0.6614811420440674], "r_norm": [0.1764272153377533]}, {"step": 5450, "epoch": 13, "train_loss": 0.001627167803235352, "cos_r_negg": [0.6598830819129944], "cos_innovation_negg": [0.6598830819129944], "cos_apical_negg": [0.6598830819129944], "cos_Ac_negg": [0.6598830819129944], "r_norm": [0.17324164509773254]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9767, "test_loss": 0.08944097690582276}, {"step": 5500, "epoch": 14, "train_loss": 0.0014794734306633472, "cos_r_negg": [0.6616568565368652], "cos_innovation_negg": [0.6616568565368652], "cos_apical_negg": [0.6616568565368652], "cos_Ac_negg": [0.6616568565368652], "r_norm": [0.17759844660758972]}, {"step": 5550, "epoch": 14, "train_loss": 0.0009711239254102111, "cos_r_negg": [0.6609523892402649], "cos_innovation_negg": [0.6609523892402649], "cos_apical_negg": [0.6609523892402649], "cos_Ac_negg": [0.6609523892402649], "r_norm": [0.17489919066429138]}, {"step": 5600, "epoch": 14, "train_loss": 0.002977547235786915, "cos_r_negg": [0.6612458229064941], "cos_innovation_negg": [0.6612458229064941], "cos_apical_negg": [0.6612458229064941], "cos_Ac_negg": [0.6612458229064941], "r_norm": [0.17257159948349]}, {"step": 5650, "epoch": 14, "train_loss": 0.0009674192988313735, "cos_r_negg": [0.6620712280273438], "cos_innovation_negg": [0.6620712280273438], "cos_apical_negg": [0.6620712280273438], "cos_Ac_negg": [0.6620712280273438], "r_norm": [0.1730516105890274]}, {"step": 5700, "epoch": 14, "train_loss": 0.0004970199661329389, "cos_r_negg": [0.6602256298065186], "cos_innovation_negg": [0.6602256298065186], "cos_apical_negg": [0.6602256298065186], "cos_Ac_negg": [0.6602256298065186], "r_norm": [0.16998662054538727]}, {"step": 5750, "epoch": 14, "train_loss": 0.002579448278993368, "cos_r_negg": [0.6612381935119629], "cos_innovation_negg": [0.6612381935119629], "cos_apical_negg": [0.6612381935119629], "cos_Ac_negg": [0.6612381935119629], "r_norm": [0.17411211133003235]}, {"step": 5800, "epoch": 14, "train_loss": 0.004208316095173359, "cos_r_negg": [0.6595847606658936], "cos_innovation_negg": [0.6595847606658936], "cos_apical_negg": [0.6595847606658936], "cos_Ac_negg": [0.6595847606658936], "r_norm": [0.18091073632240295]}, {"step": 5850, "epoch": 14, "train_loss": 0.001818808726966381, "cos_r_negg": [0.6611254215240479], "cos_innovation_negg": [0.6611254215240479], "cos_apical_negg": [0.6611254215240479], "cos_Ac_negg": [0.6611254215240479], "r_norm": [0.17458906769752502]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9763, "test_loss": 0.08962259559631347}, {"step": 5900, "epoch": 15, "train_loss": 0.001702244975604117, "cos_r_negg": [0.6622937321662903], "cos_innovation_negg": [0.6622937321662903], "cos_apical_negg": [0.6622937321662903], "cos_Ac_negg": [0.6622937321662903], "r_norm": [0.17746537923812866]}, {"step": 5950, "epoch": 15, "train_loss": 0.0025866953656077385, "cos_r_negg": [0.6620649099349976], "cos_innovation_negg": [0.6620649099349976], "cos_apical_negg": [0.6620649099349976], "cos_Ac_negg": [0.6620649099349976], "r_norm": [0.17555402219295502]}, {"step": 6000, "epoch": 15, "train_loss": 0.0031983363442122936, "cos_r_negg": [0.6604093909263611], "cos_innovation_negg": [0.6604093909263611], "cos_apical_negg": [0.6604093909263611], "cos_Ac_negg": [0.6604093909263611], "r_norm": [0.17582987248897552]}, {"step": 6050, "epoch": 15, "train_loss": 0.004987574648112059, "cos_r_negg": [0.6616648435592651], "cos_innovation_negg": [0.6616648435592651], "cos_apical_negg": [0.6616648435592651], "cos_Ac_negg": [0.6616648435592651], "r_norm": [0.17288324236869812]}, {"step": 6100, "epoch": 15, "train_loss": 0.002388033317402005, "cos_r_negg": [0.6601161956787109], "cos_innovation_negg": [0.6601161956787109], "cos_apical_negg": [0.6601161956787109], "cos_Ac_negg": [0.6601161956787109], "r_norm": [0.17045314610004425]}, {"step": 6150, "epoch": 15, "train_loss": 0.001701981294900179, "cos_r_negg": [0.6634101867675781], "cos_innovation_negg": [0.6634101867675781], "cos_apical_negg": [0.6634101867675781], "cos_Ac_negg": [0.6634101867675781], "r_norm": [0.16769972443580627]}, {"step": 6200, "epoch": 15, "train_loss": 0.0015846432652324438, "cos_r_negg": [0.6644010543823242], "cos_innovation_negg": [0.6644010543823242], "cos_apical_negg": [0.6644010543823242], "cos_Ac_negg": [0.6644010543823242], "r_norm": [0.17465442419052124]}, {"step": 6250, "epoch": 15, "train_loss": 0.0010502494405955076, "cos_r_negg": [0.6645565032958984], "cos_innovation_negg": [0.6645565032958984], "cos_apical_negg": [0.6645565032958984], "cos_Ac_negg": [0.6645565032958984], "r_norm": [0.16563507914543152]}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9774, "test_loss": 0.08891616344451904}, {"step": 6300, "epoch": 16, "train_loss": 0.002964187879115343, "cos_r_negg": [0.6626226902008057], "cos_innovation_negg": [0.6626226902008057], "cos_apical_negg": [0.6626226902008057], "cos_Ac_negg": [0.6626226902008057], "r_norm": [0.1781245470046997]}, {"step": 6350, "epoch": 16, "train_loss": 0.0007718633860349655, "cos_r_negg": [0.6630537509918213], "cos_innovation_negg": [0.6630537509918213], "cos_apical_negg": [0.6630537509918213], "cos_Ac_negg": [0.6630537509918213], "r_norm": [0.17614293098449707]}, {"step": 6400, "epoch": 16, "train_loss": 0.0027924366295337677, "cos_r_negg": [0.6617437601089478], "cos_innovation_negg": [0.6617437601089478], "cos_apical_negg": [0.6617437601089478], "cos_Ac_negg": [0.6617437601089478], "r_norm": [0.1721075177192688]}, {"step": 6450, "epoch": 16, "train_loss": 0.000959826516918838, "cos_r_negg": [0.6613703370094299], "cos_innovation_negg": [0.6613703370094299], "cos_apical_negg": [0.6613703370094299], "cos_Ac_negg": [0.6613703370094299], "r_norm": [0.176346093416214]}, {"step": 6500, "epoch": 16, "train_loss": 0.0013776915147900581, "cos_r_negg": [0.662568986415863], "cos_innovation_negg": [0.662568986415863], "cos_apical_negg": [0.662568986415863], "cos_Ac_negg": [0.662568986415863], "r_norm": [0.16943183541297913]}, {"step": 6550, "epoch": 16, "train_loss": 0.0019063070649281144, "cos_r_negg": [0.6650741100311279], "cos_innovation_negg": [0.6650741100311279], "cos_apical_negg": [0.6650741100311279], "cos_Ac_negg": [0.6650741100311279], "r_norm": [0.1697850227355957]}, {"step": 6600, "epoch": 16, "train_loss": 0.0013843283522874117, "cos_r_negg": [0.6621103286743164], "cos_innovation_negg": [0.6621103286743164], "cos_apical_negg": [0.6621103286743164], "cos_Ac_negg": [0.6621103286743164], "r_norm": [0.1635584980249405]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.977, "test_loss": 0.08964291744232178}, {"step": 6650, "epoch": 17, "train_loss": 0.0011154257226735353, "cos_r_negg": [0.6619985103607178], "cos_innovation_negg": [0.6619985103607178], "cos_apical_negg": [0.6619985103607178], "cos_Ac_negg": [0.6619985103607178], "r_norm": [0.16848492622375488]}, {"step": 6700, "epoch": 17, "train_loss": 0.0007620551041327417, "cos_r_negg": [0.6607497334480286], "cos_innovation_negg": [0.6607497334480286], "cos_apical_negg": [0.6607497334480286], "cos_Ac_negg": [0.6607497334480286], "r_norm": [0.17597457766532898]}, {"step": 6750, "epoch": 17, "train_loss": 0.0018879538401961327, "cos_r_negg": [0.6646029949188232], "cos_innovation_negg": [0.6646029949188232], "cos_apical_negg": [0.6646029949188232], "cos_Ac_negg": [0.6646029949188232], "r_norm": [0.176978200674057]}, {"step": 6800, "epoch": 17, "train_loss": 0.0020793303847312927, "cos_r_negg": [0.6638383269309998], "cos_innovation_negg": [0.6638383269309998], "cos_apical_negg": [0.6638383269309998], "cos_Ac_negg": [0.6638383269309998], "r_norm": [0.17357929050922394]}, {"step": 6850, "epoch": 17, "train_loss": 0.0012102670734748244, "cos_r_negg": [0.6635543704032898], "cos_innovation_negg": [0.6635543704032898], "cos_apical_negg": [0.6635543704032898], "cos_Ac_negg": [0.6635543704032898], "r_norm": [0.1714283972978592]}, {"step": 6900, "epoch": 17, "train_loss": 0.0018079549772664905, "cos_r_negg": [0.6638709306716919], "cos_innovation_negg": [0.6638709306716919], "cos_apical_negg": [0.6638709306716919], "cos_Ac_negg": [0.6638709306716919], "r_norm": [0.16890369355678558]}, {"step": 6950, "epoch": 17, "train_loss": 0.001843736506998539, "cos_r_negg": [0.6620208621025085], "cos_innovation_negg": [0.6620208621025085], "cos_apical_negg": [0.6620208621025085], "cos_Ac_negg": [0.6620208621025085], "r_norm": [0.17306850850582123]}, {"step": 7000, "epoch": 17, "train_loss": 0.0020277579315006733, "cos_r_negg": [0.6619917750358582], "cos_innovation_negg": [0.6619917750358582], "cos_apical_negg": [0.6619917750358582], "cos_Ac_negg": [0.6619917750358582], "r_norm": [0.16570419073104858]}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9756, "test_loss": 0.09115127239227296}, {"step": 7050, "epoch": 18, "train_loss": 0.0007009297842159867, "cos_r_negg": [0.6638060808181763], "cos_innovation_negg": [0.6638060808181763], "cos_apical_negg": [0.6638060808181763], "cos_Ac_negg": [0.6638060808181763], "r_norm": [0.1716570407152176]}, {"step": 7100, "epoch": 18, "train_loss": 0.0012758707161992788, "cos_r_negg": [0.6635274887084961], "cos_innovation_negg": [0.6635274887084961], "cos_apical_negg": [0.6635274887084961], "cos_Ac_negg": [0.6635274887084961], "r_norm": [0.17431879043579102]}, {"step": 7150, "epoch": 18, "train_loss": 0.0007745464681647718, "cos_r_negg": [0.6642131209373474], "cos_innovation_negg": [0.6642131209373474], "cos_apical_negg": [0.6642131209373474], "cos_Ac_negg": [0.6642131209373474], "r_norm": [0.17175468802452087]}, {"step": 7200, "epoch": 18, "train_loss": 0.001303626224398613, "cos_r_negg": [0.6641005277633667], "cos_innovation_negg": [0.6641005277633667], "cos_apical_negg": [0.6641005277633667], "cos_Ac_negg": [0.6641005277633667], "r_norm": [0.17121271789073944]}, {"step": 7250, "epoch": 18, "train_loss": 0.0012902661692351103, "cos_r_negg": [0.6636227369308472], "cos_innovation_negg": [0.6636227369308472], "cos_apical_negg": [0.6636227369308472], "cos_Ac_negg": [0.6636227369308472], "r_norm": [0.16889938712120056]}, {"step": 7300, "epoch": 18, "train_loss": 0.0007635349757038057, "cos_r_negg": [0.6639586687088013], "cos_innovation_negg": [0.6639586687088013], "cos_apical_negg": [0.6639586687088013], "cos_Ac_negg": [0.6639586687088013], "r_norm": [0.16924510896205902]}, {"step": 7350, "epoch": 18, "train_loss": 0.003067204263061285, "cos_r_negg": [0.6645622253417969], "cos_innovation_negg": [0.6645622253417969], "cos_apical_negg": [0.6645622253417969], "cos_Ac_negg": [0.6645622253417969], "r_norm": [0.17142267525196075]}, {"step": 7400, "epoch": 18, "train_loss": 0.0007239767583087087, "cos_r_negg": [0.6640191078186035], "cos_innovation_negg": [0.6640191078186035], "cos_apical_negg": [0.6640191078186035], "cos_Ac_negg": [0.6640191078186035], "r_norm": [0.1669405996799469]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9776, "test_loss": 0.09094000663757325}, {"step": 7450, "epoch": 19, "train_loss": 0.0006831792998127639, "cos_r_negg": [0.6619687080383301], "cos_innovation_negg": [0.6619687080383301], "cos_apical_negg": [0.6619687080383301], "cos_Ac_negg": [0.6619687080383301], "r_norm": [0.16538608074188232]}, {"step": 7500, "epoch": 19, "train_loss": 0.0010655021760612726, "cos_r_negg": [0.6635773777961731], "cos_innovation_negg": [0.6635773777961731], "cos_apical_negg": [0.6635773777961731], "cos_Ac_negg": [0.6635773777961731], "r_norm": [0.16893739998340607]}, {"step": 7550, "epoch": 19, "train_loss": 0.0006806849269196391, "cos_r_negg": [0.6641970872879028], "cos_innovation_negg": [0.6641970872879028], "cos_apical_negg": [0.6641970872879028], "cos_Ac_negg": [0.6641970872879028], "r_norm": [0.16580957174301147]}, {"step": 7600, "epoch": 19, "train_loss": 0.000440245377831161, "cos_r_negg": [0.6643680334091187], "cos_innovation_negg": [0.6643680334091187], "cos_apical_negg": [0.6643680334091187], "cos_Ac_negg": [0.6643680334091187], "r_norm": [0.17028790712356567]}, {"step": 7650, "epoch": 19, "train_loss": 0.001604728982783854, "cos_r_negg": [0.6654733419418335], "cos_innovation_negg": [0.6654733419418335], "cos_apical_negg": [0.6654733419418335], "cos_Ac_negg": [0.6654733419418335], "r_norm": [0.17399537563323975]}, {"step": 7700, "epoch": 19, "train_loss": 0.002592981094494462, "cos_r_negg": [0.6648432016372681], "cos_innovation_negg": [0.6648432016372681], "cos_apical_negg": [0.6648432016372681], "cos_Ac_negg": [0.6648432016372681], "r_norm": [0.16562968492507935]}, {"step": 7750, "epoch": 19, "train_loss": 0.0011266012443229556, "cos_r_negg": [0.6646178960800171], "cos_innovation_negg": [0.6646178960800171], "cos_apical_negg": [0.6646178960800171], "cos_Ac_negg": [0.6646178960800171], "r_norm": [0.1692628711462021]}, {"step": 7800, "epoch": 19, "train_loss": 0.0018553495174273849, "cos_r_negg": [0.6663547158241272], "cos_innovation_negg": [0.6663547158241272], "cos_apical_negg": [0.6663547158241272], "cos_Ac_negg": [0.6663547158241272], "r_norm": [0.17162495851516724]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9769, "test_loss": 0.0922179651260376}, {"step": 7850, "epoch": 20, "train_loss": 0.0014949343167245388, "cos_r_negg": [0.664222002029419], "cos_innovation_negg": [0.664222002029419], "cos_apical_negg": [0.664222002029419], "cos_Ac_negg": [0.664222002029419], "r_norm": [0.1665872037410736]}, {"step": 7900, "epoch": 20, "train_loss": 0.001578298513777554, "cos_r_negg": [0.6649748086929321], "cos_innovation_negg": [0.6649748086929321], "cos_apical_negg": [0.6649748086929321], "cos_Ac_negg": [0.6649748086929321], "r_norm": [0.1707608699798584]}, {"step": 7950, "epoch": 20, "train_loss": 0.0008258823654614389, "cos_r_negg": [0.6636658906936646], "cos_innovation_negg": [0.6636658906936646], "cos_apical_negg": [0.6636658906936646], "cos_Ac_negg": [0.6636658906936646], "r_norm": [0.17021676898002625]}, {"step": 8000, "epoch": 20, "train_loss": 0.0012938324362039566, "cos_r_negg": [0.6655979156494141], "cos_innovation_negg": [0.6655979156494141], "cos_apical_negg": [0.6655979156494141], "cos_Ac_negg": [0.6655979156494141], "r_norm": [0.16982626914978027]}, {"step": 8050, "epoch": 20, "train_loss": 0.0021154372952878475, "cos_r_negg": [0.6659412980079651], "cos_innovation_negg": [0.6659412980079651], "cos_apical_negg": [0.6659412980079651], "cos_Ac_negg": [0.6659412980079651], "r_norm": [0.17043495178222656]}, {"step": 8100, "epoch": 20, "train_loss": 0.0015141437761485577, "cos_r_negg": [0.6652356386184692], "cos_innovation_negg": [0.6652356386184692], "cos_apical_negg": [0.6652356386184692], "cos_Ac_negg": [0.6652356386184692], "r_norm": [0.16815754771232605]}, {"step": 8150, "epoch": 20, "train_loss": 0.001185504486784339, "cos_r_negg": [0.6645083427429199], "cos_innovation_negg": [0.6645083427429199], "cos_apical_negg": [0.6645083427429199], "cos_Ac_negg": [0.6645083427429199], "r_norm": [0.16986188292503357]}, {"step": 8200, "epoch": 20, "train_loss": 0.0007164583075791597, "cos_r_negg": [0.6649080514907837], "cos_innovation_negg": [0.6649080514907837], "cos_apical_negg": [0.6649080514907837], "cos_Ac_negg": [0.6649080514907837], "r_norm": [0.17069581151008606]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9764, "test_loss": 0.09235961103439332}, {"step": 8250, "epoch": 21, "train_loss": 0.0012270014267414808, "cos_r_negg": [0.6644504070281982], "cos_innovation_negg": [0.6644504070281982], "cos_apical_negg": [0.6644504070281982], "cos_Ac_negg": [0.6644504070281982], "r_norm": [0.17279984056949615]}, {"step": 8300, "epoch": 21, "train_loss": 0.0013807707000523806, "cos_r_negg": [0.6651556491851807], "cos_innovation_negg": [0.6651556491851807], "cos_apical_negg": [0.6651556491851807], "cos_Ac_negg": [0.6651556491851807], "r_norm": [0.16744640469551086]}, {"step": 8350, "epoch": 21, "train_loss": 0.0013558367500081658, "cos_r_negg": [0.6657724976539612], "cos_innovation_negg": [0.6657724976539612], "cos_apical_negg": [0.6657724976539612], "cos_Ac_negg": [0.6657724976539612], "r_norm": [0.16786617040634155]}, {"step": 8400, "epoch": 21, "train_loss": 0.0017479443922638893, "cos_r_negg": [0.6657877564430237], "cos_innovation_negg": [0.6657877564430237], "cos_apical_negg": [0.6657877564430237], "cos_Ac_negg": [0.6657877564430237], "r_norm": [0.16394180059432983]}, {"step": 8450, "epoch": 21, "train_loss": 0.001413336955010891, "cos_r_negg": [0.6661390662193298], "cos_innovation_negg": [0.6661390662193298], "cos_apical_negg": [0.6661390662193298], "cos_Ac_negg": [0.6661390662193298], "r_norm": [0.16530540585517883]}, {"step": 8500, "epoch": 21, "train_loss": 0.0015957646537572145, "cos_r_negg": [0.6660447716712952], "cos_innovation_negg": [0.6660447716712952], "cos_apical_negg": [0.6660447716712952], "cos_Ac_negg": [0.6660447716712952], "r_norm": [0.16783112287521362]}, {"step": 8550, "epoch": 21, "train_loss": 0.0010656488593667746, "cos_r_negg": [0.6666637063026428], "cos_innovation_negg": [0.6666637063026428], "cos_apical_negg": [0.6666637063026428], "cos_Ac_negg": [0.6666637063026428], "r_norm": [0.1656591147184372]}, {"step": 8600, "epoch": 21, "train_loss": 0.002240352798253298, "cos_r_negg": [0.6665427684783936], "cos_innovation_negg": [0.6665427684783936], "cos_apical_negg": [0.6665427684783936], "cos_Ac_negg": [0.6665427684783936], "r_norm": [0.1695120632648468]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.976, "test_loss": 0.09303749122619628}, {"step": 8650, "epoch": 22, "train_loss": 0.0006289861630648375, "cos_r_negg": [0.666278600692749], "cos_innovation_negg": [0.666278600692749], "cos_apical_negg": [0.666278600692749], "cos_Ac_negg": [0.666278600692749], "r_norm": [0.17096863687038422]}, {"step": 8700, "epoch": 22, "train_loss": 0.0007678767433390021, "cos_r_negg": [0.6659321188926697], "cos_innovation_negg": [0.6659321188926697], "cos_apical_negg": [0.6659321188926697], "cos_Ac_negg": [0.6659321188926697], "r_norm": [0.17015340924263]}, {"step": 8750, "epoch": 22, "train_loss": 0.0008804360404610634, "cos_r_negg": [0.6653702259063721], "cos_innovation_negg": [0.6653702259063721], "cos_apical_negg": [0.6653702259063721], "cos_Ac_negg": [0.6653702259063721], "r_norm": [0.1693313866853714]}, {"step": 8800, "epoch": 22, "train_loss": 0.0016957317711785436, "cos_r_negg": [0.6660256385803223], "cos_innovation_negg": [0.6660256385803223], "cos_apical_negg": [0.6660256385803223], "cos_Ac_negg": [0.6660256385803223], "r_norm": [0.16689372062683105]}, {"step": 8850, "epoch": 22, "train_loss": 0.0007832589326426387, "cos_r_negg": [0.6687551736831665], "cos_innovation_negg": [0.6687551736831665], "cos_apical_negg": [0.6687551736831665], "cos_Ac_negg": [0.6687551736831665], "r_norm": [0.16506756842136383]}, {"step": 8900, "epoch": 22, "train_loss": 0.002062832470983267, "cos_r_negg": [0.6666590571403503], "cos_innovation_negg": [0.6666590571403503], "cos_apical_negg": [0.6666590571403503], "cos_Ac_negg": [0.6666590571403503], "r_norm": [0.16975219547748566]}, {"step": 8950, "epoch": 22, "train_loss": 0.0009238877100870013, "cos_r_negg": [0.6662837266921997], "cos_innovation_negg": [0.6662837266921997], "cos_apical_negg": [0.6662837266921997], "cos_Ac_negg": [0.6662837266921997], "r_norm": [0.17363396286964417]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9769, "test_loss": 0.09275986576080322}, {"step": 9000, "epoch": 23, "train_loss": 0.0005426039570011199, "cos_r_negg": [0.6663780212402344], "cos_innovation_negg": [0.6663780212402344], "cos_apical_negg": [0.6663780212402344], "cos_Ac_negg": [0.6663780212402344], "r_norm": [0.16741646826267242]}, {"step": 9050, "epoch": 23, "train_loss": 0.001080094138160348, "cos_r_negg": [0.6670243740081787], "cos_innovation_negg": [0.6670243740081787], "cos_apical_negg": [0.6670243740081787], "cos_Ac_negg": [0.6670243740081787], "r_norm": [0.1692594587802887]}, {"step": 9100, "epoch": 23, "train_loss": 0.0008646490750834346, "cos_r_negg": [0.6668412685394287], "cos_innovation_negg": [0.6668412685394287], "cos_apical_negg": [0.6668412685394287], "cos_Ac_negg": [0.6668412685394287], "r_norm": [0.16468298435211182]}, {"step": 9150, "epoch": 23, "train_loss": 0.0016141673550009727, "cos_r_negg": [0.6663103103637695], "cos_innovation_negg": [0.6663103103637695], "cos_apical_negg": [0.6663103103637695], "cos_Ac_negg": [0.6663103103637695], "r_norm": [0.16689661145210266]}, {"step": 9200, "epoch": 23, "train_loss": 0.0017156414687633514, "cos_r_negg": [0.6665346622467041], "cos_innovation_negg": [0.6665346622467041], "cos_apical_negg": [0.6665346622467041], "cos_Ac_negg": [0.6665346622467041], "r_norm": [0.1663321852684021]}, {"step": 9250, "epoch": 23, "train_loss": 0.0010491367429494858, "cos_r_negg": [0.6658576726913452], "cos_innovation_negg": [0.6658576726913452], "cos_apical_negg": [0.6658576726913452], "cos_Ac_negg": [0.6658576726913452], "r_norm": [0.16685517132282257]}, {"step": 9300, "epoch": 23, "train_loss": 0.0009037284180521965, "cos_r_negg": [0.6654970049858093], "cos_innovation_negg": [0.6654970049858093], "cos_apical_negg": [0.6654970049858093], "cos_Ac_negg": [0.6654970049858093], "r_norm": [0.16875602304935455]}, {"step": 9350, "epoch": 23, "train_loss": 0.0011300718178972602, "cos_r_negg": [0.6664285063743591], "cos_innovation_negg": [0.6664285063743591], "cos_apical_negg": [0.6664285063743591], "cos_Ac_negg": [0.6664285063743591], "r_norm": [0.16760197281837463]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9763, "test_loss": 0.09372999019622803}, {"step": 9400, "epoch": 24, "train_loss": 0.0007356866262853146, "cos_r_negg": [0.6667789220809937], "cos_innovation_negg": [0.6667789220809937], "cos_apical_negg": [0.6667789220809937], "cos_Ac_negg": [0.6667789220809937], "r_norm": [0.16868163645267487]}, {"step": 9450, "epoch": 24, "train_loss": 0.0007303430465981364, "cos_r_negg": [0.6672382950782776], "cos_innovation_negg": [0.6672382950782776], "cos_apical_negg": [0.6672382950782776], "cos_Ac_negg": [0.6672382950782776], "r_norm": [0.16808977723121643]}, {"step": 9500, "epoch": 24, "train_loss": 0.0012442534789443016, "cos_r_negg": [0.6675103902816772], "cos_innovation_negg": [0.6675103902816772], "cos_apical_negg": [0.6675103902816772], "cos_Ac_negg": [0.6675103902816772], "r_norm": [0.16589809954166412]}, {"step": 9550, "epoch": 24, "train_loss": 0.0010358005529269576, "cos_r_negg": [0.667627215385437], "cos_innovation_negg": [0.667627215385437], "cos_apical_negg": [0.667627215385437], "cos_Ac_negg": [0.667627215385437], "r_norm": [0.16577830910682678]}, {"step": 9600, "epoch": 24, "train_loss": 0.0007997080683708191, "cos_r_negg": [0.6659653186798096], "cos_innovation_negg": [0.6659653186798096], "cos_apical_negg": [0.6659653186798096], "cos_Ac_negg": [0.6659653186798096], "r_norm": [0.16646955907344818]}, {"step": 9650, "epoch": 24, "train_loss": 0.0015334270428866148, "cos_r_negg": [0.6684226989746094], "cos_innovation_negg": [0.6684226989746094], "cos_apical_negg": [0.6684226989746094], "cos_Ac_negg": [0.6684226989746094], "r_norm": [0.1656925082206726]}, {"step": 9700, "epoch": 24, "train_loss": 0.0007564777042716742, "cos_r_negg": [0.6681210994720459], "cos_innovation_negg": [0.6681210994720459], "cos_apical_negg": [0.6681210994720459], "cos_Ac_negg": [0.6681210994720459], "r_norm": [0.16559502482414246]}, {"step": 9750, "epoch": 24, "train_loss": 0.0022185430862009525, "cos_r_negg": [0.6660330891609192], "cos_innovation_negg": [0.6660330891609192], "cos_apical_negg": [0.6660330891609192], "cos_Ac_negg": [0.6660330891609192], "r_norm": [0.16597822308540344]}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9767, "test_loss": 0.09350334692001343}], "final": {"test_acc": 0.9767, "test_loss": 0.09350334692001343, "wall_s": 9.961609363555908, "cos_r_negg": [0.6665401458740234], "cos_innovation_negg": [0.6665401458740234], "cos_apical_negg": [0.6665401458740234], "cos_Ac_negg": [0.6665401458740234]}} \ No newline at end of file diff --git a/results/ep_archmatch_v1_mnist_dfa_w500_d1_s1.json b/results/ep_archmatch_v1_mnist_dfa_w500_d1_s1.json new file mode 100644 index 0000000..276447d --- /dev/null +++ b/results/ep_archmatch_v1_mnist_dfa_w500_d1_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "mnist", "depth": 1, "width": 500, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_archmatch_v1_mnist_dfa_w500_d1_s1", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.5871329307556152, "cos_r_negg": [0.0007894374430179596], "cos_innovation_negg": [0.0007894374430179596], "cos_apical_negg": [0.0007894374430179596], "cos_Ac_negg": [0.0007894374430179596], "r_norm": [6.775417327880859]}, {"step": 50, "epoch": 0, "train_loss": 0.4899613857269287, "cos_r_negg": [0.43206948041915894], "cos_innovation_negg": [0.43206948041915894], "cos_apical_negg": [0.43206948041915894], "cos_Ac_negg": [0.43206948041915894], "r_norm": [1.3700265884399414]}, {"step": 100, "epoch": 0, "train_loss": 0.13584038615226746, "cos_r_negg": [0.4294331669807434], "cos_innovation_negg": [0.4294331669807434], "cos_apical_negg": [0.4294331669807434], "cos_Ac_negg": [0.4294331669807434], "r_norm": [1.096548318862915]}, {"step": 150, "epoch": 0, "train_loss": 0.2023809552192688, "cos_r_negg": [0.45162269473075867], "cos_innovation_negg": [0.45162269473075867], "cos_apical_negg": [0.45162269473075867], "cos_Ac_negg": [0.45162269473075867], "r_norm": [0.9095224142074585]}, {"step": 200, "epoch": 0, "train_loss": 0.28397268056869507, "cos_r_negg": [0.4616773724555969], "cos_innovation_negg": [0.4616773724555969], "cos_apical_negg": [0.4616773724555969], "cos_Ac_negg": [0.4616773724555969], "r_norm": [0.8920854330062866]}, {"step": 250, "epoch": 0, "train_loss": 0.3516388535499573, "cos_r_negg": [0.461736261844635], "cos_innovation_negg": [0.461736261844635], "cos_apical_negg": [0.461736261844635], "cos_Ac_negg": [0.461736261844635], "r_norm": [0.9115661978721619]}, {"step": 300, "epoch": 0, "train_loss": 0.13876698911190033, "cos_r_negg": [0.46671342849731445], "cos_innovation_negg": [0.46671342849731445], "cos_apical_negg": [0.46671342849731445], "cos_Ac_negg": [0.46671342849731445], "r_norm": [0.7718034982681274]}, {"step": 350, "epoch": 0, "train_loss": 0.20696309208869934, "cos_r_negg": [0.46781474351882935], "cos_innovation_negg": [0.46781474351882935], "cos_apical_negg": [0.46781474351882935], "cos_Ac_negg": [0.46781474351882935], "r_norm": [0.6963138580322266]}, {"epoch_end": 0, "step": 391, "test_acc": 0.9513, "test_loss": 0.16305571060180665}, {"step": 400, "epoch": 1, "train_loss": 0.14324313402175903, "cos_r_negg": [0.4829540252685547], "cos_innovation_negg": [0.4829540252685547], "cos_apical_negg": [0.4829540252685547], "cos_Ac_negg": [0.4829540252685547], "r_norm": [0.6391471028327942]}, {"step": 450, "epoch": 1, "train_loss": 0.12347893416881561, "cos_r_negg": [0.4974728226661682], "cos_innovation_negg": [0.4974728226661682], "cos_apical_negg": [0.4974728226661682], "cos_Ac_negg": [0.4974728226661682], "r_norm": [0.5809763669967651]}, {"step": 500, "epoch": 1, "train_loss": 0.1555749922990799, "cos_r_negg": [0.5127581357955933], "cos_innovation_negg": [0.5127581357955933], "cos_apical_negg": [0.5127581357955933], "cos_Ac_negg": [0.5127581357955933], "r_norm": [0.5027990341186523]}, {"step": 550, "epoch": 1, "train_loss": 0.06301746517419815, "cos_r_negg": [0.5055460333824158], "cos_innovation_negg": [0.5055460333824158], "cos_apical_negg": [0.5055460333824158], "cos_Ac_negg": [0.5055460333824158], "r_norm": [0.5517017841339111]}, {"step": 600, "epoch": 1, "train_loss": 0.04994211718440056, "cos_r_negg": [0.5179140567779541], "cos_innovation_negg": [0.5179140567779541], "cos_apical_negg": [0.5179140567779541], "cos_Ac_negg": [0.5179140567779541], "r_norm": [0.5359616279602051]}, {"step": 650, "epoch": 1, "train_loss": 0.0824631005525589, "cos_r_negg": [0.5145946741104126], "cos_innovation_negg": [0.5145946741104126], "cos_apical_negg": [0.5145946741104126], "cos_Ac_negg": [0.5145946741104126], "r_norm": [0.5243547558784485]}, {"step": 700, "epoch": 1, "train_loss": 0.11573042720556259, "cos_r_negg": [0.5178329348564148], "cos_innovation_negg": [0.5178329348564148], "cos_apical_negg": [0.5178329348564148], "cos_Ac_negg": [0.5178329348564148], "r_norm": [0.517716646194458]}, {"step": 750, "epoch": 1, "train_loss": 0.16182687878608704, "cos_r_negg": [0.5214344263076782], "cos_innovation_negg": [0.5214344263076782], "cos_apical_negg": [0.5214344263076782], "cos_Ac_negg": [0.5214344263076782], "r_norm": [0.5674691796302795]}, {"epoch_end": 1, "step": 782, "test_acc": 0.9589, "test_loss": 0.13530714874267577}, {"step": 800, "epoch": 2, "train_loss": 0.11450827866792679, "cos_r_negg": [0.5091661214828491], "cos_innovation_negg": [0.5091661214828491], "cos_apical_negg": [0.5091661214828491], "cos_Ac_negg": [0.5091661214828491], "r_norm": [0.4984610080718994]}, {"step": 850, "epoch": 2, "train_loss": 0.03856636583805084, "cos_r_negg": [0.5238624215126038], "cos_innovation_negg": [0.5238624215126038], "cos_apical_negg": [0.5238624215126038], "cos_Ac_negg": [0.5238624215126038], "r_norm": [0.435440331697464]}, {"step": 900, "epoch": 2, "train_loss": 0.06582101434469223, "cos_r_negg": [0.5383661985397339], "cos_innovation_negg": [0.5383661985397339], "cos_apical_negg": [0.5383661985397339], "cos_Ac_negg": [0.5383661985397339], "r_norm": [0.4646889567375183]}, {"step": 950, "epoch": 2, "train_loss": 0.12663978338241577, "cos_r_negg": [0.53855299949646], "cos_innovation_negg": [0.53855299949646], "cos_apical_negg": [0.53855299949646], "cos_Ac_negg": [0.53855299949646], "r_norm": [0.3788195252418518]}, {"step": 1000, "epoch": 2, "train_loss": 0.07001999020576477, "cos_r_negg": [0.5356705188751221], "cos_innovation_negg": [0.5356705188751221], "cos_apical_negg": [0.5356705188751221], "cos_Ac_negg": [0.5356705188751221], "r_norm": [0.39627984166145325]}, {"step": 1050, "epoch": 2, "train_loss": 0.04979276657104492, "cos_r_negg": [0.5428125262260437], "cos_innovation_negg": [0.5428125262260437], "cos_apical_negg": [0.5428125262260437], "cos_Ac_negg": [0.5428125262260437], "r_norm": [0.4240158200263977]}, {"step": 1100, "epoch": 2, "train_loss": 0.10400481522083282, "cos_r_negg": [0.5325610637664795], "cos_innovation_negg": [0.5325610637664795], "cos_apical_negg": [0.5325610637664795], "cos_Ac_negg": [0.5325610637664795], "r_norm": [0.3773163855075836]}, {"step": 1150, "epoch": 2, "train_loss": 0.09066778421401978, "cos_r_negg": [0.5591815710067749], "cos_innovation_negg": [0.5591815710067749], "cos_apical_negg": [0.5591815710067749], "cos_Ac_negg": [0.5591815710067749], "r_norm": [0.34616532921791077]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9671, "test_loss": 0.1076137861251831}, {"step": 1200, "epoch": 3, "train_loss": 0.0663936659693718, "cos_r_negg": [0.5473842024803162], "cos_innovation_negg": [0.5473842024803162], "cos_apical_negg": [0.5473842024803162], "cos_Ac_negg": [0.5473842024803162], "r_norm": [0.44569331407546997]}, {"step": 1250, "epoch": 3, "train_loss": 0.08653033524751663, "cos_r_negg": [0.5647534728050232], "cos_innovation_negg": [0.5647534728050232], "cos_apical_negg": [0.5647534728050232], "cos_Ac_negg": [0.5647534728050232], "r_norm": [0.3562757968902588]}, {"step": 1300, "epoch": 3, "train_loss": 0.017032869160175323, "cos_r_negg": [0.5569448471069336], "cos_innovation_negg": [0.5569448471069336], "cos_apical_negg": [0.5569448471069336], "cos_Ac_negg": [0.5569448471069336], "r_norm": [0.3674372434616089]}, {"step": 1350, "epoch": 3, "train_loss": 0.041269972920417786, "cos_r_negg": [0.565823495388031], "cos_innovation_negg": [0.565823495388031], "cos_apical_negg": [0.565823495388031], "cos_Ac_negg": [0.565823495388031], "r_norm": [0.3499387502670288]}, {"step": 1400, "epoch": 3, "train_loss": 0.04396485537290573, "cos_r_negg": [0.5667727589607239], "cos_innovation_negg": [0.5667727589607239], "cos_apical_negg": [0.5667727589607239], "cos_Ac_negg": [0.5667727589607239], "r_norm": [0.3593904376029968]}, {"step": 1450, "epoch": 3, "train_loss": 0.01968386583030224, "cos_r_negg": [0.5770570635795593], "cos_innovation_negg": [0.5770570635795593], "cos_apical_negg": [0.5770570635795593], "cos_Ac_negg": [0.5770570635795593], "r_norm": [0.3280967175960541]}, {"step": 1500, "epoch": 3, "train_loss": 0.11142823100090027, "cos_r_negg": [0.5879610776901245], "cos_innovation_negg": [0.5879610776901245], "cos_apical_negg": [0.5879610776901245], "cos_Ac_negg": [0.5879610776901245], "r_norm": [0.2886162996292114]}, {"step": 1550, "epoch": 3, "train_loss": 0.03138778358697891, "cos_r_negg": [0.5852259397506714], "cos_innovation_negg": [0.5852259397506714], "cos_apical_negg": [0.5852259397506714], "cos_Ac_negg": [0.5852259397506714], "r_norm": [0.32250094413757324]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9699, "test_loss": 0.10255150547027587}, {"step": 1600, "epoch": 4, "train_loss": 0.05445222556591034, "cos_r_negg": [0.5850704908370972], "cos_innovation_negg": [0.5850704908370972], "cos_apical_negg": [0.5850704908370972], "cos_Ac_negg": [0.5850704908370972], "r_norm": [0.31904006004333496]}, {"step": 1650, "epoch": 4, "train_loss": 0.05383739247918129, "cos_r_negg": [0.5897184014320374], "cos_innovation_negg": [0.5897184014320374], "cos_apical_negg": [0.5897184014320374], "cos_Ac_negg": [0.5897184014320374], "r_norm": [0.29477283358573914]}, {"step": 1700, "epoch": 4, "train_loss": 0.036422692239284515, "cos_r_negg": [0.6006355285644531], "cos_innovation_negg": [0.6006355285644531], "cos_apical_negg": [0.6006355285644531], "cos_Ac_negg": [0.6006355285644531], "r_norm": [0.3245965838432312]}, {"step": 1750, "epoch": 4, "train_loss": 0.037037335336208344, "cos_r_negg": [0.5937342643737793], "cos_innovation_negg": [0.5937342643737793], "cos_apical_negg": [0.5937342643737793], "cos_Ac_negg": [0.5937342643737793], "r_norm": [0.2968626618385315]}, {"step": 1800, "epoch": 4, "train_loss": 0.05671168863773346, "cos_r_negg": [0.5913489460945129], "cos_innovation_negg": [0.5913489460945129], "cos_apical_negg": [0.5913489460945129], "cos_Ac_negg": [0.5913489460945129], "r_norm": [0.3239508867263794]}, {"step": 1850, "epoch": 4, "train_loss": 0.0898481234908104, "cos_r_negg": [0.599265992641449], "cos_innovation_negg": [0.599265992641449], "cos_apical_negg": [0.599265992641449], "cos_Ac_negg": [0.599265992641449], "r_norm": [0.31218409538269043]}, {"step": 1900, "epoch": 4, "train_loss": 0.013059291988611221, "cos_r_negg": [0.6088038682937622], "cos_innovation_negg": [0.6088038682937622], "cos_apical_negg": [0.6088038682937622], "cos_Ac_negg": [0.6088038682937622], "r_norm": [0.3217884302139282]}, {"step": 1950, "epoch": 4, "train_loss": 0.10430076718330383, "cos_r_negg": [0.606650173664093], "cos_innovation_negg": [0.606650173664093], "cos_apical_negg": [0.606650173664093], "cos_Ac_negg": [0.606650173664093], "r_norm": [0.30629342794418335]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9734, "test_loss": 0.09389463729858398}, {"step": 2000, "epoch": 5, "train_loss": 0.022511960938572884, "cos_r_negg": [0.6163021326065063], "cos_innovation_negg": [0.6163021326065063], "cos_apical_negg": [0.6163021326065063], "cos_Ac_negg": [0.6163021326065063], "r_norm": [0.31790637969970703]}, {"step": 2050, "epoch": 5, "train_loss": 0.02383582293987274, "cos_r_negg": [0.6130483746528625], "cos_innovation_negg": [0.6130483746528625], "cos_apical_negg": [0.6130483746528625], "cos_Ac_negg": [0.6130483746528625], "r_norm": [0.32056406140327454]}, {"step": 2100, "epoch": 5, "train_loss": 0.030350230634212494, "cos_r_negg": [0.6171742677688599], "cos_innovation_negg": [0.6171742677688599], "cos_apical_negg": [0.6171742677688599], "cos_Ac_negg": [0.6171742677688599], "r_norm": [0.29953497648239136]}, {"step": 2150, "epoch": 5, "train_loss": 0.009370625019073486, "cos_r_negg": [0.6154868602752686], "cos_innovation_negg": [0.6154868602752686], "cos_apical_negg": [0.6154868602752686], "cos_Ac_negg": [0.6154868602752686], "r_norm": [0.2886853516101837]}, {"step": 2200, "epoch": 5, "train_loss": 0.03889497369527817, "cos_r_negg": [0.6149801015853882], "cos_innovation_negg": [0.6149801015853882], "cos_apical_negg": [0.6149801015853882], "cos_Ac_negg": [0.6149801015853882], "r_norm": [0.28372272849082947]}, {"step": 2250, "epoch": 5, "train_loss": 0.012745029293000698, "cos_r_negg": [0.6196362972259521], "cos_innovation_negg": [0.6196362972259521], "cos_apical_negg": [0.6196362972259521], "cos_Ac_negg": [0.6196362972259521], "r_norm": [0.26498618721961975]}, {"step": 2300, "epoch": 5, "train_loss": 0.019418878480792046, "cos_r_negg": [0.6155492663383484], "cos_innovation_negg": [0.6155492663383484], "cos_apical_negg": [0.6155492663383484], "cos_Ac_negg": [0.6155492663383484], "r_norm": [0.2581597864627838]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9741, "test_loss": 0.09270588188171387}, {"step": 2350, "epoch": 6, "train_loss": 0.005637042224407196, "cos_r_negg": [0.6193826198577881], "cos_innovation_negg": [0.6193826198577881], "cos_apical_negg": [0.6193826198577881], "cos_Ac_negg": [0.6193826198577881], "r_norm": [0.24440300464630127]}, {"step": 2400, "epoch": 6, "train_loss": 0.0077408985234797, "cos_r_negg": [0.625503420829773], "cos_innovation_negg": [0.625503420829773], "cos_apical_negg": [0.625503420829773], "cos_Ac_negg": [0.625503420829773], "r_norm": [0.25606435537338257]}, {"step": 2450, "epoch": 6, "train_loss": 0.007536623626947403, "cos_r_negg": [0.6215605139732361], "cos_innovation_negg": [0.6215605139732361], "cos_apical_negg": [0.6215605139732361], "cos_Ac_negg": [0.6215605139732361], "r_norm": [0.27582263946533203]}, {"step": 2500, "epoch": 6, "train_loss": 0.01037085521966219, "cos_r_negg": [0.6233494877815247], "cos_innovation_negg": [0.6233494877815247], "cos_apical_negg": [0.6233494877815247], "cos_Ac_negg": [0.6233494877815247], "r_norm": [0.24718007445335388]}, {"step": 2550, "epoch": 6, "train_loss": 0.007006144151091576, "cos_r_negg": [0.6211683750152588], "cos_innovation_negg": [0.6211683750152588], "cos_apical_negg": [0.6211683750152588], "cos_Ac_negg": [0.6211683750152588], "r_norm": [0.252031147480011]}, {"step": 2600, "epoch": 6, "train_loss": 0.007235796190798283, "cos_r_negg": [0.6243923306465149], "cos_innovation_negg": [0.6243923306465149], "cos_apical_negg": [0.6243923306465149], "cos_Ac_negg": [0.6243923306465149], "r_norm": [0.26205599308013916]}, {"step": 2650, "epoch": 6, "train_loss": 0.020719125866889954, "cos_r_negg": [0.6301621794700623], "cos_innovation_negg": [0.6301621794700623], "cos_apical_negg": [0.6301621794700623], "cos_Ac_negg": [0.6301621794700623], "r_norm": [0.2830565869808197]}, {"step": 2700, "epoch": 6, "train_loss": 0.006858041509985924, "cos_r_negg": [0.6301217079162598], "cos_innovation_negg": [0.6301217079162598], "cos_apical_negg": [0.6301217079162598], "cos_Ac_negg": [0.6301217079162598], "r_norm": [0.27750077843666077]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.973, "test_loss": 0.09121898431777954}, {"step": 2750, "epoch": 7, "train_loss": 0.004343417938798666, "cos_r_negg": [0.6310592293739319], "cos_innovation_negg": [0.6310592293739319], "cos_apical_negg": [0.6310592293739319], "cos_Ac_negg": [0.6310592293739319], "r_norm": [0.24817146360874176]}, {"step": 2800, "epoch": 7, "train_loss": 0.002654121955856681, "cos_r_negg": [0.6374483108520508], "cos_innovation_negg": [0.6374483108520508], "cos_apical_negg": [0.6374483108520508], "cos_Ac_negg": [0.6374483108520508], "r_norm": [0.2316819727420807]}, {"step": 2850, "epoch": 7, "train_loss": 0.004564232192933559, "cos_r_negg": [0.6348337531089783], "cos_innovation_negg": [0.6348337531089783], "cos_apical_negg": [0.6348337531089783], "cos_Ac_negg": [0.6348337531089783], "r_norm": [0.24878667294979095]}, {"step": 2900, "epoch": 7, "train_loss": 0.005149048287421465, "cos_r_negg": [0.6364458799362183], "cos_innovation_negg": [0.6364458799362183], "cos_apical_negg": [0.6364458799362183], "cos_Ac_negg": [0.6364458799362183], "r_norm": [0.21879738569259644]}, {"step": 2950, "epoch": 7, "train_loss": 0.0020277039147913456, "cos_r_negg": [0.6381431818008423], "cos_innovation_negg": [0.6381431818008423], "cos_apical_negg": [0.6381431818008423], "cos_Ac_negg": [0.6381431818008423], "r_norm": [0.23765629529953003]}, {"step": 3000, "epoch": 7, "train_loss": 0.00639483705163002, "cos_r_negg": [0.6367685794830322], "cos_innovation_negg": [0.6367685794830322], "cos_apical_negg": [0.6367685794830322], "cos_Ac_negg": [0.6367685794830322], "r_norm": [0.2400972694158554]}, {"step": 3050, "epoch": 7, "train_loss": 0.005372028797864914, "cos_r_negg": [0.6392872929573059], "cos_innovation_negg": [0.6392872929573059], "cos_apical_negg": [0.6392872929573059], "cos_Ac_negg": [0.6392872929573059], "r_norm": [0.2362382560968399]}, {"step": 3100, "epoch": 7, "train_loss": 0.005397685803472996, "cos_r_negg": [0.643151581287384], "cos_innovation_negg": [0.643151581287384], "cos_apical_negg": [0.643151581287384], "cos_Ac_negg": [0.643151581287384], "r_norm": [0.2216721624135971]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9745, "test_loss": 0.09312191104888916}, {"step": 3150, "epoch": 8, "train_loss": 0.0036268564872443676, "cos_r_negg": [0.642423152923584], "cos_innovation_negg": [0.642423152923584], "cos_apical_negg": [0.642423152923584], "cos_Ac_negg": [0.642423152923584], "r_norm": [0.23231104016304016]}, {"step": 3200, "epoch": 8, "train_loss": 0.005781321786344051, "cos_r_negg": [0.6426788568496704], "cos_innovation_negg": [0.6426788568496704], "cos_apical_negg": [0.6426788568496704], "cos_Ac_negg": [0.6426788568496704], "r_norm": [0.22894644737243652]}, {"step": 3250, "epoch": 8, "train_loss": 0.0024674476590007544, "cos_r_negg": [0.6440749764442444], "cos_innovation_negg": [0.6440749764442444], "cos_apical_negg": [0.6440749764442444], "cos_Ac_negg": [0.6440749764442444], "r_norm": [0.22774353623390198]}, {"step": 3300, "epoch": 8, "train_loss": 0.0033361695241183043, "cos_r_negg": [0.6411113739013672], "cos_innovation_negg": [0.6411113739013672], "cos_apical_negg": [0.6411113739013672], "cos_Ac_negg": [0.6411113739013672], "r_norm": [0.241564080119133]}, {"step": 3350, "epoch": 8, "train_loss": 0.007605422288179398, "cos_r_negg": [0.645399808883667], "cos_innovation_negg": [0.645399808883667], "cos_apical_negg": [0.645399808883667], "cos_Ac_negg": [0.645399808883667], "r_norm": [0.21616588532924652]}, {"step": 3400, "epoch": 8, "train_loss": 0.005649367347359657, "cos_r_negg": [0.6431790590286255], "cos_innovation_negg": [0.6431790590286255], "cos_apical_negg": [0.6431790590286255], "cos_Ac_negg": [0.6431790590286255], "r_norm": [0.22725659608840942]}, {"step": 3450, "epoch": 8, "train_loss": 0.010687895119190216, "cos_r_negg": [0.6437346339225769], "cos_innovation_negg": [0.6437346339225769], "cos_apical_negg": [0.6437346339225769], "cos_Ac_negg": [0.6437346339225769], "r_norm": [0.22498571872711182]}, {"step": 3500, "epoch": 8, "train_loss": 0.008275466971099377, "cos_r_negg": [0.6459589004516602], "cos_innovation_negg": [0.6459589004516602], "cos_apical_negg": [0.6459589004516602], "cos_Ac_negg": [0.6459589004516602], "r_norm": [0.21755841374397278]}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9739, "test_loss": 0.09261880922317504}, {"step": 3550, "epoch": 9, "train_loss": 0.0033913725055754185, "cos_r_negg": [0.64422607421875], "cos_innovation_negg": [0.64422607421875], "cos_apical_negg": [0.64422607421875], "cos_Ac_negg": [0.64422607421875], "r_norm": [0.21306243538856506]}, {"step": 3600, "epoch": 9, "train_loss": 0.0037876181304454803, "cos_r_negg": [0.6462929248809814], "cos_innovation_negg": [0.6462929248809814], "cos_apical_negg": [0.6462929248809814], "cos_Ac_negg": [0.6462929248809814], "r_norm": [0.2055078148841858]}, {"step": 3650, "epoch": 9, "train_loss": 0.004807349294424057, "cos_r_negg": [0.6475275158882141], "cos_innovation_negg": [0.6475275158882141], "cos_apical_negg": [0.6475275158882141], "cos_Ac_negg": [0.6475275158882141], "r_norm": [0.20297284424304962]}, {"step": 3700, "epoch": 9, "train_loss": 0.006330684758722782, "cos_r_negg": [0.6445780992507935], "cos_innovation_negg": [0.6445780992507935], "cos_apical_negg": [0.6445780992507935], "cos_Ac_negg": [0.6445780992507935], "r_norm": [0.217559352517128]}, {"step": 3750, "epoch": 9, "train_loss": 0.0048070065677165985, "cos_r_negg": [0.6473070979118347], "cos_innovation_negg": [0.6473070979118347], "cos_apical_negg": [0.6473070979118347], "cos_Ac_negg": [0.6473070979118347], "r_norm": [0.20688438415527344]}, {"step": 3800, "epoch": 9, "train_loss": 0.002280543092638254, "cos_r_negg": [0.6484519839286804], "cos_innovation_negg": [0.6484519839286804], "cos_apical_negg": [0.6484519839286804], "cos_Ac_negg": [0.6484519839286804], "r_norm": [0.22260627150535583]}, {"step": 3850, "epoch": 9, "train_loss": 0.0034050147514790297, "cos_r_negg": [0.6502039432525635], "cos_innovation_negg": [0.6502039432525635], "cos_apical_negg": [0.6502039432525635], "cos_Ac_negg": [0.6502039432525635], "r_norm": [0.23101477324962616]}, {"step": 3900, "epoch": 9, "train_loss": 0.009064959362149239, "cos_r_negg": [0.6498516798019409], "cos_innovation_negg": [0.6498516798019409], "cos_apical_negg": [0.6498516798019409], "cos_Ac_negg": [0.6498516798019409], "r_norm": [0.20993450284004211]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9754, "test_loss": 0.0931414834022522}, {"step": 3950, "epoch": 10, "train_loss": 0.003206011839210987, "cos_r_negg": [0.6483080983161926], "cos_innovation_negg": [0.6483080983161926], "cos_apical_negg": [0.6483080983161926], "cos_Ac_negg": [0.6483080983161926], "r_norm": [0.22909995913505554]}, {"step": 4000, "epoch": 10, "train_loss": 0.0030084254685789347, "cos_r_negg": [0.6503021121025085], "cos_innovation_negg": [0.6503021121025085], "cos_apical_negg": [0.6503021121025085], "cos_Ac_negg": [0.6503021121025085], "r_norm": [0.22354793548583984]}, {"step": 4050, "epoch": 10, "train_loss": 0.0023085027933120728, "cos_r_negg": [0.6498113870620728], "cos_innovation_negg": [0.6498113870620728], "cos_apical_negg": [0.6498113870620728], "cos_Ac_negg": [0.6498113870620728], "r_norm": [0.22366786003112793]}, {"step": 4100, "epoch": 10, "train_loss": 0.0026850071735680103, "cos_r_negg": [0.6519977450370789], "cos_innovation_negg": [0.6519977450370789], "cos_apical_negg": [0.6519977450370789], "cos_Ac_negg": [0.6519977450370789], "r_norm": [0.2167125642299652]}, {"step": 4150, "epoch": 10, "train_loss": 0.001762542175129056, "cos_r_negg": [0.6522499322891235], "cos_innovation_negg": [0.6522499322891235], "cos_apical_negg": [0.6522499322891235], "cos_Ac_negg": [0.6522499322891235], "r_norm": [0.20908883213996887]}, {"step": 4200, "epoch": 10, "train_loss": 0.003125084564089775, "cos_r_negg": [0.652247965335846], "cos_innovation_negg": [0.652247965335846], "cos_apical_negg": [0.652247965335846], "cos_Ac_negg": [0.652247965335846], "r_norm": [0.20513826608657837]}, {"step": 4250, "epoch": 10, "train_loss": 0.004066352732479572, "cos_r_negg": [0.6537549495697021], "cos_innovation_negg": [0.6537549495697021], "cos_apical_negg": [0.6537549495697021], "cos_Ac_negg": [0.6537549495697021], "r_norm": [0.21182428300380707]}, {"step": 4300, "epoch": 10, "train_loss": 0.0026898474898189306, "cos_r_negg": [0.6533263921737671], "cos_innovation_negg": [0.6533263921737671], "cos_apical_negg": [0.6533263921737671], "cos_Ac_negg": [0.6533263921737671], "r_norm": [0.21003782749176025]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9746, "test_loss": 0.09517458992004395}, {"step": 4350, "epoch": 11, "train_loss": 0.00444463174790144, "cos_r_negg": [0.6540101766586304], "cos_innovation_negg": [0.6540101766586304], "cos_apical_negg": [0.6540101766586304], "cos_Ac_negg": [0.6540101766586304], "r_norm": [0.20803695917129517]}, {"step": 4400, "epoch": 11, "train_loss": 0.0021628979593515396, "cos_r_negg": [0.6541988253593445], "cos_innovation_negg": [0.6541988253593445], "cos_apical_negg": [0.6541988253593445], "cos_Ac_negg": [0.6541988253593445], "r_norm": [0.2045953869819641]}, {"step": 4450, "epoch": 11, "train_loss": 0.0030792183242738247, "cos_r_negg": [0.6544393301010132], "cos_innovation_negg": [0.6544393301010132], "cos_apical_negg": [0.6544393301010132], "cos_Ac_negg": [0.6544393301010132], "r_norm": [0.20447349548339844]}, {"step": 4500, "epoch": 11, "train_loss": 0.0009226790280081332, "cos_r_negg": [0.655596137046814], "cos_innovation_negg": [0.655596137046814], "cos_apical_negg": [0.655596137046814], "cos_Ac_negg": [0.655596137046814], "r_norm": [0.21108298003673553]}, {"step": 4550, "epoch": 11, "train_loss": 0.004318329505622387, "cos_r_negg": [0.6546974182128906], "cos_innovation_negg": [0.6546974182128906], "cos_apical_negg": [0.6546974182128906], "cos_Ac_negg": [0.6546974182128906], "r_norm": [0.20992915332317352]}, {"step": 4600, "epoch": 11, "train_loss": 0.0013384392950683832, "cos_r_negg": [0.6563288569450378], "cos_innovation_negg": [0.6563288569450378], "cos_apical_negg": [0.6563288569450378], "cos_Ac_negg": [0.6563288569450378], "r_norm": [0.20001664757728577]}, {"step": 4650, "epoch": 11, "train_loss": 0.001995312748476863, "cos_r_negg": [0.6583000421524048], "cos_innovation_negg": [0.6583000421524048], "cos_apical_negg": [0.6583000421524048], "cos_Ac_negg": [0.6583000421524048], "r_norm": [0.20129242539405823]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9751, "test_loss": 0.09585179243087769}, {"step": 4700, "epoch": 12, "train_loss": 0.00276106595993042, "cos_r_negg": [0.6551060676574707], "cos_innovation_negg": [0.6551060676574707], "cos_apical_negg": [0.6551060676574707], "cos_Ac_negg": [0.6551060676574707], "r_norm": [0.2051093876361847]}, {"step": 4750, "epoch": 12, "train_loss": 0.0021925249602645636, "cos_r_negg": [0.6564251780509949], "cos_innovation_negg": [0.6564251780509949], "cos_apical_negg": [0.6564251780509949], "cos_Ac_negg": [0.6564251780509949], "r_norm": [0.20655840635299683]}, {"step": 4800, "epoch": 12, "train_loss": 0.0033762066159397364, "cos_r_negg": [0.6560477018356323], "cos_innovation_negg": [0.6560477018356323], "cos_apical_negg": [0.6560477018356323], "cos_Ac_negg": [0.6560477018356323], "r_norm": [0.20325595140457153]}, {"step": 4850, "epoch": 12, "train_loss": 0.0017550287302583456, "cos_r_negg": [0.6573566198348999], "cos_innovation_negg": [0.6573566198348999], "cos_apical_negg": [0.6573566198348999], "cos_Ac_negg": [0.6573566198348999], "r_norm": [0.2027435153722763]}, {"step": 4900, "epoch": 12, "train_loss": 0.0019733579829335213, "cos_r_negg": [0.6561962366104126], "cos_innovation_negg": [0.6561962366104126], "cos_apical_negg": [0.6561962366104126], "cos_Ac_negg": [0.6561962366104126], "r_norm": [0.21009036898612976]}, {"step": 4950, "epoch": 12, "train_loss": 0.001979782246053219, "cos_r_negg": [0.6579755544662476], "cos_innovation_negg": [0.6579755544662476], "cos_apical_negg": [0.6579755544662476], "cos_Ac_negg": [0.6579755544662476], "r_norm": [0.2035580724477768]}, {"step": 5000, "epoch": 12, "train_loss": 0.0017296983860433102, "cos_r_negg": [0.6566193103790283], "cos_innovation_negg": [0.6566193103790283], "cos_apical_negg": [0.6566193103790283], "cos_Ac_negg": [0.6566193103790283], "r_norm": [0.1996634155511856]}, {"step": 5050, "epoch": 12, "train_loss": 0.0031489452812820673, "cos_r_negg": [0.6580419540405273], "cos_innovation_negg": [0.6580419540405273], "cos_apical_negg": [0.6580419540405273], "cos_Ac_negg": [0.6580419540405273], "r_norm": [0.20000208914279938]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9753, "test_loss": 0.09754808311462403}, {"step": 5100, "epoch": 13, "train_loss": 0.001423270907253027, "cos_r_negg": [0.6583412289619446], "cos_innovation_negg": [0.6583412289619446], "cos_apical_negg": [0.6583412289619446], "cos_Ac_negg": [0.6583412289619446], "r_norm": [0.20267890393733978]}, {"step": 5150, "epoch": 13, "train_loss": 0.003079029731452465, "cos_r_negg": [0.6600550413131714], "cos_innovation_negg": [0.6600550413131714], "cos_apical_negg": [0.6600550413131714], "cos_Ac_negg": [0.6600550413131714], "r_norm": [0.20320561528205872]}, {"step": 5200, "epoch": 13, "train_loss": 0.002190121216699481, "cos_r_negg": [0.659995436668396], "cos_innovation_negg": [0.659995436668396], "cos_apical_negg": [0.659995436668396], "cos_Ac_negg": [0.659995436668396], "r_norm": [0.20398621261119843]}, {"step": 5250, "epoch": 13, "train_loss": 0.0011460487730801105, "cos_r_negg": [0.658480703830719], "cos_innovation_negg": [0.658480703830719], "cos_apical_negg": [0.658480703830719], "cos_Ac_negg": [0.658480703830719], "r_norm": [0.20022249221801758]}, {"step": 5300, "epoch": 13, "train_loss": 0.002707799430936575, "cos_r_negg": [0.6594223380088806], "cos_innovation_negg": [0.6594223380088806], "cos_apical_negg": [0.6594223380088806], "cos_Ac_negg": [0.6594223380088806], "r_norm": [0.2002583146095276]}, {"step": 5350, "epoch": 13, "train_loss": 0.002040266525000334, "cos_r_negg": [0.6596497297286987], "cos_innovation_negg": [0.6596497297286987], "cos_apical_negg": [0.6596497297286987], "cos_Ac_negg": [0.6596497297286987], "r_norm": [0.2047087848186493]}, {"step": 5400, "epoch": 13, "train_loss": 0.0027785291895270348, "cos_r_negg": [0.661243200302124], "cos_innovation_negg": [0.661243200302124], "cos_apical_negg": [0.661243200302124], "cos_Ac_negg": [0.661243200302124], "r_norm": [0.2006371021270752]}, {"step": 5450, "epoch": 13, "train_loss": 0.0014241302851587534, "cos_r_negg": [0.6602023243904114], "cos_innovation_negg": [0.6602023243904114], "cos_apical_negg": [0.6602023243904114], "cos_Ac_negg": [0.6602023243904114], "r_norm": [0.19457554817199707]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9754, "test_loss": 0.09738770227432252}, {"step": 5500, "epoch": 14, "train_loss": 0.0013688614126294851, "cos_r_negg": [0.6591587066650391], "cos_innovation_negg": [0.6591587066650391], "cos_apical_negg": [0.6591587066650391], "cos_Ac_negg": [0.6591587066650391], "r_norm": [0.20840312540531158]}, {"step": 5550, "epoch": 14, "train_loss": 0.0009603003854863346, "cos_r_negg": [0.660380482673645], "cos_innovation_negg": [0.660380482673645], "cos_apical_negg": [0.660380482673645], "cos_Ac_negg": [0.660380482673645], "r_norm": [0.19567610323429108]}, {"step": 5600, "epoch": 14, "train_loss": 0.002555398503318429, "cos_r_negg": [0.6614912152290344], "cos_innovation_negg": [0.6614912152290344], "cos_apical_negg": [0.6614912152290344], "cos_Ac_negg": [0.6614912152290344], "r_norm": [0.2004864662885666]}, {"step": 5650, "epoch": 14, "train_loss": 0.0016400362364947796, "cos_r_negg": [0.6613389253616333], "cos_innovation_negg": [0.6613389253616333], "cos_apical_negg": [0.6613389253616333], "cos_Ac_negg": [0.6613389253616333], "r_norm": [0.19445368647575378]}, {"step": 5700, "epoch": 14, "train_loss": 0.0004286013136152178, "cos_r_negg": [0.6616840958595276], "cos_innovation_negg": [0.6616840958595276], "cos_apical_negg": [0.6616840958595276], "cos_Ac_negg": [0.6616840958595276], "r_norm": [0.1951034963130951]}, {"step": 5750, "epoch": 14, "train_loss": 0.0019340405706316233, "cos_r_negg": [0.6610091924667358], "cos_innovation_negg": [0.6610091924667358], "cos_apical_negg": [0.6610091924667358], "cos_Ac_negg": [0.6610091924667358], "r_norm": [0.20009282231330872]}, {"step": 5800, "epoch": 14, "train_loss": 0.003140780609101057, "cos_r_negg": [0.6600695848464966], "cos_innovation_negg": [0.6600695848464966], "cos_apical_negg": [0.6600695848464966], "cos_Ac_negg": [0.6600695848464966], "r_norm": [0.1963227093219757]}, {"step": 5850, "epoch": 14, "train_loss": 0.0019042978528887033, "cos_r_negg": [0.6614630222320557], "cos_innovation_negg": [0.6614630222320557], "cos_apical_negg": [0.6614630222320557], "cos_Ac_negg": [0.6614630222320557], "r_norm": [0.19571933150291443]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.975, "test_loss": 0.0972707181930542}, {"step": 5900, "epoch": 15, "train_loss": 0.001635093241930008, "cos_r_negg": [0.6617323160171509], "cos_innovation_negg": [0.6617323160171509], "cos_apical_negg": [0.6617323160171509], "cos_Ac_negg": [0.6617323160171509], "r_norm": [0.19847707450389862]}, {"step": 5950, "epoch": 15, "train_loss": 0.00196370598860085, "cos_r_negg": [0.6621772646903992], "cos_innovation_negg": [0.6621772646903992], "cos_apical_negg": [0.6621772646903992], "cos_Ac_negg": [0.6621772646903992], "r_norm": [0.19559775292873383]}, {"step": 6000, "epoch": 15, "train_loss": 0.002638794481754303, "cos_r_negg": [0.6623073220252991], "cos_innovation_negg": [0.6623073220252991], "cos_apical_negg": [0.6623073220252991], "cos_Ac_negg": [0.6623073220252991], "r_norm": [0.19585786759853363]}, {"step": 6050, "epoch": 15, "train_loss": 0.0034665423445403576, "cos_r_negg": [0.66242516040802], "cos_innovation_negg": [0.66242516040802], "cos_apical_negg": [0.66242516040802], "cos_Ac_negg": [0.66242516040802], "r_norm": [0.19371157884597778]}, {"step": 6100, "epoch": 15, "train_loss": 0.0015658899210393429, "cos_r_negg": [0.6632505059242249], "cos_innovation_negg": [0.6632505059242249], "cos_apical_negg": [0.6632505059242249], "cos_Ac_negg": [0.6632505059242249], "r_norm": [0.1963942050933838]}, {"step": 6150, "epoch": 15, "train_loss": 0.001391019904986024, "cos_r_negg": [0.664225697517395], "cos_innovation_negg": [0.664225697517395], "cos_apical_negg": [0.664225697517395], "cos_Ac_negg": [0.664225697517395], "r_norm": [0.19110608100891113]}, {"step": 6200, "epoch": 15, "train_loss": 0.00141964724753052, "cos_r_negg": [0.6635614037513733], "cos_innovation_negg": [0.6635614037513733], "cos_apical_negg": [0.6635614037513733], "cos_Ac_negg": [0.6635614037513733], "r_norm": [0.19873687624931335]}, {"step": 6250, "epoch": 15, "train_loss": 0.0012938386062160134, "cos_r_negg": [0.6637795567512512], "cos_innovation_negg": [0.6637795567512512], "cos_apical_negg": [0.6637795567512512], "cos_Ac_negg": [0.6637795567512512], "r_norm": [0.18979889154434204]}, {"epoch_end": 15, "step": 6256, "test_acc": 0.975, "test_loss": 0.09868302183151245}, {"step": 6300, "epoch": 16, "train_loss": 0.003708784468472004, "cos_r_negg": [0.6645128130912781], "cos_innovation_negg": [0.6645128130912781], "cos_apical_negg": [0.6645128130912781], "cos_Ac_negg": [0.6645128130912781], "r_norm": [0.1947154998779297]}, {"step": 6350, "epoch": 16, "train_loss": 0.0009563872590661049, "cos_r_negg": [0.6640611886978149], "cos_innovation_negg": [0.6640611886978149], "cos_apical_negg": [0.6640611886978149], "cos_Ac_negg": [0.6640611886978149], "r_norm": [0.1945975422859192]}, {"step": 6400, "epoch": 16, "train_loss": 0.0012989963870495558, "cos_r_negg": [0.6640341281890869], "cos_innovation_negg": [0.6640341281890869], "cos_apical_negg": [0.6640341281890869], "cos_Ac_negg": [0.6640341281890869], "r_norm": [0.19316376745700836]}, {"step": 6450, "epoch": 16, "train_loss": 0.0009034154354594648, "cos_r_negg": [0.6640784740447998], "cos_innovation_negg": [0.6640784740447998], "cos_apical_negg": [0.6640784740447998], "cos_Ac_negg": [0.6640784740447998], "r_norm": [0.19596627354621887]}, {"step": 6500, "epoch": 16, "train_loss": 0.0011173357488587499, "cos_r_negg": [0.6653001308441162], "cos_innovation_negg": [0.6653001308441162], "cos_apical_negg": [0.6653001308441162], "cos_Ac_negg": [0.6653001308441162], "r_norm": [0.19167789816856384]}, {"step": 6550, "epoch": 16, "train_loss": 0.0017973369685932994, "cos_r_negg": [0.6657671332359314], "cos_innovation_negg": [0.6657671332359314], "cos_apical_negg": [0.6657671332359314], "cos_Ac_negg": [0.6657671332359314], "r_norm": [0.19383355975151062]}, {"step": 6600, "epoch": 16, "train_loss": 0.0014234530972316861, "cos_r_negg": [0.6659306883811951], "cos_innovation_negg": [0.6659306883811951], "cos_apical_negg": [0.6659306883811951], "cos_Ac_negg": [0.6659306883811951], "r_norm": [0.18624195456504822]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9751, "test_loss": 0.09878506841659546}, {"step": 6650, "epoch": 17, "train_loss": 0.0014292561681941152, "cos_r_negg": [0.6655169129371643], "cos_innovation_negg": [0.6655169129371643], "cos_apical_negg": [0.6655169129371643], "cos_Ac_negg": [0.6655169129371643], "r_norm": [0.19217170774936676]}, {"step": 6700, "epoch": 17, "train_loss": 0.0005652151303365827, "cos_r_negg": [0.6649637222290039], "cos_innovation_negg": [0.6649637222290039], "cos_apical_negg": [0.6649637222290039], "cos_Ac_negg": [0.6649637222290039], "r_norm": [0.19252251088619232]}, {"step": 6750, "epoch": 17, "train_loss": 0.0017452200409024954, "cos_r_negg": [0.6652095317840576], "cos_innovation_negg": [0.6652095317840576], "cos_apical_negg": [0.6652095317840576], "cos_Ac_negg": [0.6652095317840576], "r_norm": [0.1888444423675537]}, {"step": 6800, "epoch": 17, "train_loss": 0.0021325349807739258, "cos_r_negg": [0.6658817529678345], "cos_innovation_negg": [0.6658817529678345], "cos_apical_negg": [0.6658817529678345], "cos_Ac_negg": [0.6658817529678345], "r_norm": [0.18643619120121002]}, {"step": 6850, "epoch": 17, "train_loss": 0.0012312886537984014, "cos_r_negg": [0.6662223935127258], "cos_innovation_negg": [0.6662223935127258], "cos_apical_negg": [0.6662223935127258], "cos_Ac_negg": [0.6662223935127258], "r_norm": [0.18658147752285004]}, {"step": 6900, "epoch": 17, "train_loss": 0.0027486728504300117, "cos_r_negg": [0.6672632694244385], "cos_innovation_negg": [0.6672632694244385], "cos_apical_negg": [0.6672632694244385], "cos_Ac_negg": [0.6672632694244385], "r_norm": [0.19228437542915344]}, {"step": 6950, "epoch": 17, "train_loss": 0.0019218033412471414, "cos_r_negg": [0.667006254196167], "cos_innovation_negg": [0.667006254196167], "cos_apical_negg": [0.667006254196167], "cos_Ac_negg": [0.667006254196167], "r_norm": [0.19100181758403778]}, {"step": 7000, "epoch": 17, "train_loss": 0.002351178787648678, "cos_r_negg": [0.6653410196304321], "cos_innovation_negg": [0.6653410196304321], "cos_apical_negg": [0.6653410196304321], "cos_Ac_negg": [0.6653410196304321], "r_norm": [0.19082188606262207]}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9762, "test_loss": 0.0997799391746521}, {"step": 7050, "epoch": 18, "train_loss": 0.000601451494731009, "cos_r_negg": [0.6676850318908691], "cos_innovation_negg": [0.6676850318908691], "cos_apical_negg": [0.6676850318908691], "cos_Ac_negg": [0.6676850318908691], "r_norm": [0.19441576302051544]}, {"step": 7100, "epoch": 18, "train_loss": 0.0018468634225428104, "cos_r_negg": [0.6673315763473511], "cos_innovation_negg": [0.6673315763473511], "cos_apical_negg": [0.6673315763473511], "cos_Ac_negg": [0.6673315763473511], "r_norm": [0.1975240409374237]}, {"step": 7150, "epoch": 18, "train_loss": 0.0006448308122344315, "cos_r_negg": [0.6680618524551392], "cos_innovation_negg": [0.6680618524551392], "cos_apical_negg": [0.6680618524551392], "cos_Ac_negg": [0.6680618524551392], "r_norm": [0.19087910652160645]}, {"step": 7200, "epoch": 18, "train_loss": 0.001669116085395217, "cos_r_negg": [0.6676591634750366], "cos_innovation_negg": [0.6676591634750366], "cos_apical_negg": [0.6676591634750366], "cos_Ac_negg": [0.6676591634750366], "r_norm": [0.1889713704586029]}, {"step": 7250, "epoch": 18, "train_loss": 0.0016679498367011547, "cos_r_negg": [0.6677061319351196], "cos_innovation_negg": [0.6677061319351196], "cos_apical_negg": [0.6677061319351196], "cos_Ac_negg": [0.6677061319351196], "r_norm": [0.18926844000816345]}, {"step": 7300, "epoch": 18, "train_loss": 0.000765252741985023, "cos_r_negg": [0.6677827835083008], "cos_innovation_negg": [0.6677827835083008], "cos_apical_negg": [0.6677827835083008], "cos_Ac_negg": [0.6677827835083008], "r_norm": [0.1881237030029297]}, {"step": 7350, "epoch": 18, "train_loss": 0.0019045684020966291, "cos_r_negg": [0.6677905321121216], "cos_innovation_negg": [0.6677905321121216], "cos_apical_negg": [0.6677905321121216], "cos_Ac_negg": [0.6677905321121216], "r_norm": [0.1860121786594391]}, {"step": 7400, "epoch": 18, "train_loss": 0.0004984107217751443, "cos_r_negg": [0.6677904725074768], "cos_innovation_negg": [0.6677904725074768], "cos_apical_negg": [0.6677904725074768], "cos_Ac_negg": [0.6677904725074768], "r_norm": [0.18569402396678925]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9756, "test_loss": 0.10035949821472168}, {"step": 7450, "epoch": 19, "train_loss": 0.000881918182130903, "cos_r_negg": [0.666430652141571], "cos_innovation_negg": [0.666430652141571], "cos_apical_negg": [0.666430652141571], "cos_Ac_negg": [0.666430652141571], "r_norm": [0.18551306426525116]}, {"step": 7500, "epoch": 19, "train_loss": 0.0010666967136785388, "cos_r_negg": [0.6663866639137268], "cos_innovation_negg": [0.6663866639137268], "cos_apical_negg": [0.6663866639137268], "cos_Ac_negg": [0.6663866639137268], "r_norm": [0.18741640448570251]}, {"step": 7550, "epoch": 19, "train_loss": 0.0005765580572187901, "cos_r_negg": [0.6664718389511108], "cos_innovation_negg": [0.6664718389511108], "cos_apical_negg": [0.6664718389511108], "cos_Ac_negg": [0.6664718389511108], "r_norm": [0.1862810254096985]}, {"step": 7600, "epoch": 19, "train_loss": 0.0007428928511217237, "cos_r_negg": [0.6669197082519531], "cos_innovation_negg": [0.6669197082519531], "cos_apical_negg": [0.6669197082519531], "cos_Ac_negg": [0.6669197082519531], "r_norm": [0.18796692788600922]}, {"step": 7650, "epoch": 19, "train_loss": 0.0012970201205462217, "cos_r_negg": [0.6691609025001526], "cos_innovation_negg": [0.6691609025001526], "cos_apical_negg": [0.6691609025001526], "cos_Ac_negg": [0.6691609025001526], "r_norm": [0.1898581087589264]}, {"step": 7700, "epoch": 19, "train_loss": 0.0021766116842627525, "cos_r_negg": [0.6688386797904968], "cos_innovation_negg": [0.6688386797904968], "cos_apical_negg": [0.6688386797904968], "cos_Ac_negg": [0.6688386797904968], "r_norm": [0.1896129548549652]}, {"step": 7750, "epoch": 19, "train_loss": 0.0013679638504981995, "cos_r_negg": [0.6675209999084473], "cos_innovation_negg": [0.6675209999084473], "cos_apical_negg": [0.6675209999084473], "cos_Ac_negg": [0.6675209999084473], "r_norm": [0.1880582720041275]}, {"step": 7800, "epoch": 19, "train_loss": 0.001310515683144331, "cos_r_negg": [0.6670775413513184], "cos_innovation_negg": [0.6670775413513184], "cos_apical_negg": [0.6670775413513184], "cos_Ac_negg": [0.6670775413513184], "r_norm": [0.19043809175491333]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9756, "test_loss": 0.10122288036346436}, {"step": 7850, "epoch": 20, "train_loss": 0.0017029824666678905, "cos_r_negg": [0.667471706867218], "cos_innovation_negg": [0.667471706867218], "cos_apical_negg": [0.667471706867218], "cos_Ac_negg": [0.667471706867218], "r_norm": [0.18382754921913147]}, {"step": 7900, "epoch": 20, "train_loss": 0.0012194182490929961, "cos_r_negg": [0.6680639982223511], "cos_innovation_negg": [0.6680639982223511], "cos_apical_negg": [0.6680639982223511], "cos_Ac_negg": [0.6680639982223511], "r_norm": [0.18649348616600037]}, {"step": 7950, "epoch": 20, "train_loss": 0.00088641291949898, "cos_r_negg": [0.6669683456420898], "cos_innovation_negg": [0.6669683456420898], "cos_apical_negg": [0.6669683456420898], "cos_Ac_negg": [0.6669683456420898], "r_norm": [0.18541556596755981]}, {"step": 8000, "epoch": 20, "train_loss": 0.0012518854346126318, "cos_r_negg": [0.6674504280090332], "cos_innovation_negg": [0.6674504280090332], "cos_apical_negg": [0.6674504280090332], "cos_Ac_negg": [0.6674504280090332], "r_norm": [0.18614190816879272]}, {"step": 8050, "epoch": 20, "train_loss": 0.0019545434042811394, "cos_r_negg": [0.6684018969535828], "cos_innovation_negg": [0.6684018969535828], "cos_apical_negg": [0.6684018969535828], "cos_Ac_negg": [0.6684018969535828], "r_norm": [0.1847819685935974]}, {"step": 8100, "epoch": 20, "train_loss": 0.0011930239852517843, "cos_r_negg": [0.6683642864227295], "cos_innovation_negg": [0.6683642864227295], "cos_apical_negg": [0.6683642864227295], "cos_Ac_negg": [0.6683642864227295], "r_norm": [0.18506300449371338]}, {"step": 8150, "epoch": 20, "train_loss": 0.001413327525369823, "cos_r_negg": [0.668941080570221], "cos_innovation_negg": [0.668941080570221], "cos_apical_negg": [0.668941080570221], "cos_Ac_negg": [0.668941080570221], "r_norm": [0.18788985908031464]}, {"step": 8200, "epoch": 20, "train_loss": 0.001157931168563664, "cos_r_negg": [0.6699210405349731], "cos_innovation_negg": [0.6699210405349731], "cos_apical_negg": [0.6699210405349731], "cos_Ac_negg": [0.6699210405349731], "r_norm": [0.1897650510072708]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9757, "test_loss": 0.1011201545715332}, {"step": 8250, "epoch": 21, "train_loss": 0.0010319348657503724, "cos_r_negg": [0.6689672470092773], "cos_innovation_negg": [0.6689672470092773], "cos_apical_negg": [0.6689672470092773], "cos_Ac_negg": [0.6689672470092773], "r_norm": [0.18865783512592316]}, {"step": 8300, "epoch": 21, "train_loss": 0.001167097594588995, "cos_r_negg": [0.6686141490936279], "cos_innovation_negg": [0.6686141490936279], "cos_apical_negg": [0.6686141490936279], "cos_Ac_negg": [0.6686141490936279], "r_norm": [0.18496745824813843]}, {"step": 8350, "epoch": 21, "train_loss": 0.0013086674734950066, "cos_r_negg": [0.6695606112480164], "cos_innovation_negg": [0.6695606112480164], "cos_apical_negg": [0.6695606112480164], "cos_Ac_negg": [0.6695606112480164], "r_norm": [0.18640708923339844]}, {"step": 8400, "epoch": 21, "train_loss": 0.0009835525415837765, "cos_r_negg": [0.6693465709686279], "cos_innovation_negg": [0.6693465709686279], "cos_apical_negg": [0.6693465709686279], "cos_Ac_negg": [0.6693465709686279], "r_norm": [0.18348956108093262]}, {"step": 8450, "epoch": 21, "train_loss": 0.0011744195362553, "cos_r_negg": [0.6694566011428833], "cos_innovation_negg": [0.6694566011428833], "cos_apical_negg": [0.6694566011428833], "cos_Ac_negg": [0.6694566011428833], "r_norm": [0.18191364407539368]}, {"step": 8500, "epoch": 21, "train_loss": 0.00111324992030859, "cos_r_negg": [0.6705453991889954], "cos_innovation_negg": [0.6705453991889954], "cos_apical_negg": [0.6705453991889954], "cos_Ac_negg": [0.6705453991889954], "r_norm": [0.1867508888244629]}, {"step": 8550, "epoch": 21, "train_loss": 0.0011014826595783234, "cos_r_negg": [0.6698875427246094], "cos_innovation_negg": [0.6698875427246094], "cos_apical_negg": [0.6698875427246094], "cos_Ac_negg": [0.6698875427246094], "r_norm": [0.18466798961162567]}, {"step": 8600, "epoch": 21, "train_loss": 0.0022371753584593534, "cos_r_negg": [0.6699787378311157], "cos_innovation_negg": [0.6699787378311157], "cos_apical_negg": [0.6699787378311157], "cos_Ac_negg": [0.6699787378311157], "r_norm": [0.18645025789737701]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9751, "test_loss": 0.10225288105010986}, {"step": 8650, "epoch": 22, "train_loss": 0.000763885211199522, "cos_r_negg": [0.6693414449691772], "cos_innovation_negg": [0.6693414449691772], "cos_apical_negg": [0.6693414449691772], "cos_Ac_negg": [0.6693414449691772], "r_norm": [0.19102288782596588]}, {"step": 8700, "epoch": 22, "train_loss": 0.0007907305844128132, "cos_r_negg": [0.6701774597167969], "cos_innovation_negg": [0.6701774597167969], "cos_apical_negg": [0.6701774597167969], "cos_Ac_negg": [0.6701774597167969], "r_norm": [0.18649140000343323]}, {"step": 8750, "epoch": 22, "train_loss": 0.0009433200466446579, "cos_r_negg": [0.6706094741821289], "cos_innovation_negg": [0.6706094741821289], "cos_apical_negg": [0.6706094741821289], "cos_Ac_negg": [0.6706094741821289], "r_norm": [0.1870427429676056]}, {"step": 8800, "epoch": 22, "train_loss": 0.000975463364738971, "cos_r_negg": [0.6702993512153625], "cos_innovation_negg": [0.6702993512153625], "cos_apical_negg": [0.6702993512153625], "cos_Ac_negg": [0.6702993512153625], "r_norm": [0.18265846371650696]}, {"step": 8850, "epoch": 22, "train_loss": 0.001242354977875948, "cos_r_negg": [0.6702213287353516], "cos_innovation_negg": [0.6702213287353516], "cos_apical_negg": [0.6702213287353516], "cos_Ac_negg": [0.6702213287353516], "r_norm": [0.18444925546646118]}, {"step": 8900, "epoch": 22, "train_loss": 0.0017221624730154872, "cos_r_negg": [0.6695889234542847], "cos_innovation_negg": [0.6695889234542847], "cos_apical_negg": [0.6695889234542847], "cos_Ac_negg": [0.6695889234542847], "r_norm": [0.18694347143173218]}, {"step": 8950, "epoch": 22, "train_loss": 0.0009166446980088949, "cos_r_negg": [0.6699918508529663], "cos_innovation_negg": [0.6699918508529663], "cos_apical_negg": [0.6699918508529663], "cos_Ac_negg": [0.6699918508529663], "r_norm": [0.1874484121799469]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9751, "test_loss": 0.10232902002334594}, {"step": 9000, "epoch": 23, "train_loss": 0.0006901266751810908, "cos_r_negg": [0.670666515827179], "cos_innovation_negg": [0.670666515827179], "cos_apical_negg": [0.670666515827179], "cos_Ac_negg": [0.670666515827179], "r_norm": [0.18466335535049438]}, {"step": 9050, "epoch": 23, "train_loss": 0.0012942194007337093, "cos_r_negg": [0.671380877494812], "cos_innovation_negg": [0.671380877494812], "cos_apical_negg": [0.671380877494812], "cos_Ac_negg": [0.671380877494812], "r_norm": [0.18487384915351868]}, {"step": 9100, "epoch": 23, "train_loss": 0.001055062049999833, "cos_r_negg": [0.6716364622116089], "cos_innovation_negg": [0.6716364622116089], "cos_apical_negg": [0.6716364622116089], "cos_Ac_negg": [0.6716364622116089], "r_norm": [0.18265020847320557]}, {"step": 9150, "epoch": 23, "train_loss": 0.0013636950170621276, "cos_r_negg": [0.671456515789032], "cos_innovation_negg": [0.671456515789032], "cos_apical_negg": [0.671456515789032], "cos_Ac_negg": [0.671456515789032], "r_norm": [0.1858920454978943]}, {"step": 9200, "epoch": 23, "train_loss": 0.0016949005657806993, "cos_r_negg": [0.6722328662872314], "cos_innovation_negg": [0.6722328662872314], "cos_apical_negg": [0.6722328662872314], "cos_Ac_negg": [0.6722328662872314], "r_norm": [0.18604162335395813]}, {"step": 9250, "epoch": 23, "train_loss": 0.0008038675296120346, "cos_r_negg": [0.6715922355651855], "cos_innovation_negg": [0.6715922355651855], "cos_apical_negg": [0.6715922355651855], "cos_Ac_negg": [0.6715922355651855], "r_norm": [0.18438726663589478]}, {"step": 9300, "epoch": 23, "train_loss": 0.0010811779648065567, "cos_r_negg": [0.6712080836296082], "cos_innovation_negg": [0.6712080836296082], "cos_apical_negg": [0.6712080836296082], "cos_Ac_negg": [0.6712080836296082], "r_norm": [0.1827498972415924]}, {"step": 9350, "epoch": 23, "train_loss": 0.0008483516285195947, "cos_r_negg": [0.6726276874542236], "cos_innovation_negg": [0.6726276874542236], "cos_apical_negg": [0.6726276874542236], "cos_Ac_negg": [0.6726276874542236], "r_norm": [0.1805519163608551]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9757, "test_loss": 0.10322239894866943}, {"step": 9400, "epoch": 24, "train_loss": 0.0007654058281332254, "cos_r_negg": [0.6711655855178833], "cos_innovation_negg": [0.6711655855178833], "cos_apical_negg": [0.6711655855178833], "cos_Ac_negg": [0.6711655855178833], "r_norm": [0.1830424964427948]}, {"step": 9450, "epoch": 24, "train_loss": 0.000545462709851563, "cos_r_negg": [0.6720145344734192], "cos_innovation_negg": [0.6720145344734192], "cos_apical_negg": [0.6720145344734192], "cos_Ac_negg": [0.6720145344734192], "r_norm": [0.18395021557807922]}, {"step": 9500, "epoch": 24, "train_loss": 0.0009387051686644554, "cos_r_negg": [0.671858549118042], "cos_innovation_negg": [0.671858549118042], "cos_apical_negg": [0.671858549118042], "cos_Ac_negg": [0.671858549118042], "r_norm": [0.1816437840461731]}, {"step": 9550, "epoch": 24, "train_loss": 0.0009216958424076438, "cos_r_negg": [0.6720470786094666], "cos_innovation_negg": [0.6720470786094666], "cos_apical_negg": [0.6720470786094666], "cos_Ac_negg": [0.6720470786094666], "r_norm": [0.1831769198179245]}, {"step": 9600, "epoch": 24, "train_loss": 0.0008054062491282821, "cos_r_negg": [0.6722546219825745], "cos_innovation_negg": [0.6722546219825745], "cos_apical_negg": [0.6722546219825745], "cos_Ac_negg": [0.6722546219825745], "r_norm": [0.18059439957141876]}, {"step": 9650, "epoch": 24, "train_loss": 0.0013229851610958576, "cos_r_negg": [0.6718299388885498], "cos_innovation_negg": [0.6718299388885498], "cos_apical_negg": [0.6718299388885498], "cos_Ac_negg": [0.6718299388885498], "r_norm": [0.1806354820728302]}, {"step": 9700, "epoch": 24, "train_loss": 0.0008755388553254306, "cos_r_negg": [0.6715790033340454], "cos_innovation_negg": [0.6715790033340454], "cos_apical_negg": [0.6715790033340454], "cos_Ac_negg": [0.6715790033340454], "r_norm": [0.18285630643367767]}, {"step": 9750, "epoch": 24, "train_loss": 0.002127270679920912, "cos_r_negg": [0.6713520884513855], "cos_innovation_negg": [0.6713520884513855], "cos_apical_negg": [0.6713520884513855], "cos_Ac_negg": [0.6713520884513855], "r_norm": [0.18298110365867615]}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9757, "test_loss": 0.10320457763671875}], "final": {"test_acc": 0.9757, "test_loss": 0.10320457763671875, "wall_s": 9.812992572784424, "cos_r_negg": [0.6719449758529663], "cos_innovation_negg": [0.6719449758529663], "cos_apical_negg": [0.6719449758529663], "cos_Ac_negg": [0.6719449758529663]}} \ No newline at end of file diff --git a/results/ep_archmatch_v1_mnist_sdil_w500_d1_s0.json b/results/ep_archmatch_v1_mnist_sdil_w500_d1_s0.json new file mode 100644 index 0000000..6ed0676 --- /dev/null +++ b/results/ep_archmatch_v1_mnist_sdil_w500_d1_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 1, "width": 500, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_archmatch_v1_mnist_sdil_w500_d1_s0", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.529653549194336, "cos_r_negg": [0.008219992741942406], "cos_innovation_negg": [0.008219992741942406], "cos_apical_negg": [0.008219992741942406], "cos_Ac_negg": [0.008219992741942406], "r_norm": [6.905431747436523], "ldr": {"dL_sdil": -0.18514204025268555, "dL_bp": -0.46199512481689453, "ratio": 0.4007445756620572}}, {"step": 50, "epoch": 0, "train_loss": 0.5371271371841431, "cos_r_negg": [0.44773948192596436], "cos_innovation_negg": [0.44773948192596436], "cos_apical_negg": [0.44773948192596436], "cos_Ac_negg": [0.44773948192596436], "r_norm": [1.4367446899414062]}, {"step": 100, "epoch": 0, "train_loss": 0.15707623958587646, "cos_r_negg": [0.46268290281295776], "cos_innovation_negg": [0.46268290281295776], "cos_apical_negg": [0.46268290281295776], "cos_Ac_negg": [0.46268290281295776], "r_norm": [1.1784470081329346]}, {"step": 150, "epoch": 0, "train_loss": 0.17869894206523895, "cos_r_negg": [0.47690844535827637], "cos_innovation_negg": [0.47690844535827637], "cos_apical_negg": [0.47690844535827637], "cos_Ac_negg": [0.47690844535827637], "r_norm": [0.9737180471420288]}, {"step": 200, "epoch": 0, "train_loss": 0.2869090437889099, "cos_r_negg": [0.47075355052948], "cos_innovation_negg": [0.47075355052948], "cos_apical_negg": [0.47075355052948], "cos_Ac_negg": [0.47075355052948], "r_norm": [0.9603960514068604]}, {"step": 250, "epoch": 0, "train_loss": 0.28400173783302307, "cos_r_negg": [0.4832213521003723], "cos_innovation_negg": [0.4832213521003723], "cos_apical_negg": [0.4832213521003723], "cos_Ac_negg": [0.4832213521003723], "r_norm": [0.8602281212806702], "ldr": {"dL_sdil": -0.018383294343948364, "dL_bp": -0.016621455550193787, "ratio": 1.105997864533232}}, {"step": 300, "epoch": 0, "train_loss": 0.12333448976278305, "cos_r_negg": [0.4824344217777252], "cos_innovation_negg": [0.4824344217777252], "cos_apical_negg": [0.4824344217777252], "cos_Ac_negg": [0.4824344217777252], "r_norm": [0.7302887439727783]}, {"step": 350, "epoch": 0, "train_loss": 0.19479742646217346, "cos_r_negg": [0.5052607655525208], "cos_innovation_negg": [0.5052607655525208], "cos_apical_negg": [0.5052607655525208], "cos_Ac_negg": [0.5052607655525208], "r_norm": [0.6253390908241272]}, {"epoch_end": 0, "step": 391, "test_acc": 0.954, "test_loss": 0.15761138916015624}, {"step": 400, "epoch": 1, "train_loss": 0.13774877786636353, "cos_r_negg": [0.5046982765197754], "cos_innovation_negg": [0.5046982765197754], "cos_apical_negg": [0.5046982765197754], "cos_Ac_negg": [0.5046982765197754], "r_norm": [0.6836100816726685]}, {"step": 450, "epoch": 1, "train_loss": 0.14885926246643066, "cos_r_negg": [0.5095247030258179], "cos_innovation_negg": [0.5095247030258179], "cos_apical_negg": [0.5095247030258179], "cos_Ac_negg": [0.5095247030258179], "r_norm": [0.5687665343284607]}, {"step": 500, "epoch": 1, "train_loss": 0.1716763824224472, "cos_r_negg": [0.5296971797943115], "cos_innovation_negg": [0.5296971797943115], "cos_apical_negg": [0.5296971797943115], "cos_Ac_negg": [0.5296971797943115], "r_norm": [0.5095346570014954], "ldr": {"dL_sdil": -0.007089674472808838, "dL_bp": -0.006206996738910675, "ratio": 1.142206895061632}}, {"step": 550, "epoch": 1, "train_loss": 0.09458061307668686, "cos_r_negg": [0.5323493480682373], "cos_innovation_negg": [0.5323493480682373], "cos_apical_negg": [0.5323493480682373], "cos_Ac_negg": [0.5323493480682373], "r_norm": [0.6105064153671265]}, {"step": 600, "epoch": 1, "train_loss": 0.05246064066886902, "cos_r_negg": [0.5408417582511902], "cos_innovation_negg": [0.5408417582511902], "cos_apical_negg": [0.5408417582511902], "cos_Ac_negg": [0.5408417582511902], "r_norm": [0.49598371982574463]}, {"step": 650, "epoch": 1, "train_loss": 0.06112802028656006, "cos_r_negg": [0.5437751412391663], "cos_innovation_negg": [0.5437751412391663], "cos_apical_negg": [0.5437751412391663], "cos_Ac_negg": [0.5437751412391663], "r_norm": [0.5059418678283691]}, {"step": 700, "epoch": 1, "train_loss": 0.10810565203428268, "cos_r_negg": [0.5459623336791992], "cos_innovation_negg": [0.5459623336791992], "cos_apical_negg": [0.5459623336791992], "cos_Ac_negg": [0.5459623336791992], "r_norm": [0.4998188614845276]}, {"step": 750, "epoch": 1, "train_loss": 0.15891008079051971, "cos_r_negg": [0.5564911365509033], "cos_innovation_negg": [0.5564911365509033], "cos_apical_negg": [0.5564911365509033], "cos_Ac_negg": [0.5564911365509033], "r_norm": [0.6518878936767578], "ldr": {"dL_sdil": -0.024759113788604736, "dL_bp": -0.022933632135391235, "ratio": 1.0795984535915013}}, {"epoch_end": 1, "step": 782, "test_acc": 0.9533, "test_loss": 0.1500544303894043}, {"step": 800, "epoch": 2, "train_loss": 0.09059587121009827, "cos_r_negg": [0.5440113544464111], "cos_innovation_negg": [0.5440113544464111], "cos_apical_negg": [0.5440113544464111], "cos_Ac_negg": [0.5440113544464111], "r_norm": [0.4640093445777893]}, {"step": 850, "epoch": 2, "train_loss": 0.02787810005247593, "cos_r_negg": [0.5659802556037903], "cos_innovation_negg": [0.5659802556037903], "cos_apical_negg": [0.5659802556037903], "cos_Ac_negg": [0.5659802556037903], "r_norm": [0.4173128008842468]}, {"step": 900, "epoch": 2, "train_loss": 0.0723535567522049, "cos_r_negg": [0.5670180320739746], "cos_innovation_negg": [0.5670180320739746], "cos_apical_negg": [0.5670180320739746], "cos_Ac_negg": [0.5670180320739746], "r_norm": [0.4639837145805359]}, {"step": 950, "epoch": 2, "train_loss": 0.09411744773387909, "cos_r_negg": [0.5666851997375488], "cos_innovation_negg": [0.5666851997375488], "cos_apical_negg": [0.5666851997375488], "cos_Ac_negg": [0.5666851997375488], "r_norm": [0.32857054471969604]}, {"step": 1000, "epoch": 2, "train_loss": 0.07529803365468979, "cos_r_negg": [0.5711220502853394], "cos_innovation_negg": [0.5711220502853394], "cos_apical_negg": [0.5711220502853394], "cos_Ac_negg": [0.5711220502853394], "r_norm": [0.34924834966659546], "ldr": {"dL_sdil": -0.004454225301742554, "dL_bp": -0.0038444846868515015, "ratio": 1.158601389927868}}, {"step": 1050, "epoch": 2, "train_loss": 0.03930548205971718, "cos_r_negg": [0.5628199577331543], "cos_innovation_negg": [0.5628199577331543], "cos_apical_negg": [0.5628199577331543], "cos_Ac_negg": [0.5628199577331543], "r_norm": [0.3775678873062134]}, {"step": 1100, "epoch": 2, "train_loss": 0.12777133285999298, "cos_r_negg": [0.5679937601089478], "cos_innovation_negg": [0.5679937601089478], "cos_apical_negg": [0.5679937601089478], "cos_Ac_negg": [0.5679937601089478], "r_norm": [0.38171401619911194]}, {"step": 1150, "epoch": 2, "train_loss": 0.09708690643310547, "cos_r_negg": [0.5808869004249573], "cos_innovation_negg": [0.5808869004249573], "cos_apical_negg": [0.5808869004249573], "cos_Ac_negg": [0.5808869004249573], "r_norm": [0.25966185331344604]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9706, "test_loss": 0.0982839433670044}, {"step": 1200, "epoch": 3, "train_loss": 0.046243295073509216, "cos_r_negg": [0.5773853063583374], "cos_innovation_negg": [0.5773853063583374], "cos_apical_negg": [0.5773853063583374], "cos_Ac_negg": [0.5773853063583374], "r_norm": [0.3021707832813263]}, {"step": 1250, "epoch": 3, "train_loss": 0.053017839789390564, "cos_r_negg": [0.59014493227005], "cos_innovation_negg": [0.59014493227005], "cos_apical_negg": [0.59014493227005], "cos_Ac_negg": [0.59014493227005], "r_norm": [0.3062102496623993], "ldr": {"dL_sdil": -0.004912190139293671, "dL_bp": -0.004440739750862122, "ratio": 1.1061648317355284}}, {"step": 1300, "epoch": 3, "train_loss": 0.03673747181892395, "cos_r_negg": [0.5901116132736206], "cos_innovation_negg": [0.5901116132736206], "cos_apical_negg": [0.5901116132736206], "cos_Ac_negg": [0.5901116132736206], "r_norm": [0.2799038290977478]}, {"step": 1350, "epoch": 3, "train_loss": 0.027321703732013702, "cos_r_negg": [0.5893756151199341], "cos_innovation_negg": [0.5893756151199341], "cos_apical_negg": [0.5893756151199341], "cos_Ac_negg": [0.5893756151199341], "r_norm": [0.3704427182674408]}, {"step": 1400, "epoch": 3, "train_loss": 0.03065216727554798, "cos_r_negg": [0.5927216410636902], "cos_innovation_negg": [0.5927216410636902], "cos_apical_negg": [0.5927216410636902], "cos_Ac_negg": [0.5927216410636902], "r_norm": [0.36008816957473755]}, {"step": 1450, "epoch": 3, "train_loss": 0.04428139328956604, "cos_r_negg": [0.6035908460617065], "cos_innovation_negg": [0.6035908460617065], "cos_apical_negg": [0.6035908460617065], "cos_Ac_negg": [0.6035908460617065], "r_norm": [0.3728402554988861]}, {"step": 1500, "epoch": 3, "train_loss": 0.07053318619728088, "cos_r_negg": [0.6108893156051636], "cos_innovation_negg": [0.6108893156051636], "cos_apical_negg": [0.6108893156051636], "cos_Ac_negg": [0.6108893156051636], "r_norm": [0.26522207260131836], "ldr": {"dL_sdil": -0.003581516444683075, "dL_bp": -0.0031932219862937927, "ratio": 1.1215995818818583}}, {"step": 1550, "epoch": 3, "train_loss": 0.05413009226322174, "cos_r_negg": [0.6020864844322205], "cos_innovation_negg": [0.6020864844322205], "cos_apical_negg": [0.6020864844322205], "cos_Ac_negg": [0.6020864844322205], "r_norm": [0.29422637820243835]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9671, "test_loss": 0.11090864219665528}, {"step": 1600, "epoch": 4, "train_loss": 0.03308474272489548, "cos_r_negg": [0.6081780195236206], "cos_innovation_negg": [0.6081780195236206], "cos_apical_negg": [0.6081780195236206], "cos_Ac_negg": [0.6081780195236206], "r_norm": [0.2861747145652771]}, {"step": 1650, "epoch": 4, "train_loss": 0.08432745933532715, "cos_r_negg": [0.6038168668746948], "cos_innovation_negg": [0.6038168668746948], "cos_apical_negg": [0.6038168668746948], "cos_Ac_negg": [0.6038168668746948], "r_norm": [0.2860708236694336]}, {"step": 1700, "epoch": 4, "train_loss": 0.060790859162807465, "cos_r_negg": [0.6114307045936584], "cos_innovation_negg": [0.6114307045936584], "cos_apical_negg": [0.6114307045936584], "cos_Ac_negg": [0.6114307045936584], "r_norm": [0.26344746351242065]}, {"step": 1750, "epoch": 4, "train_loss": 0.05738605558872223, "cos_r_negg": [0.6160759925842285], "cos_innovation_negg": [0.6160759925842285], "cos_apical_negg": [0.6160759925842285], "cos_Ac_negg": [0.6160759925842285], "r_norm": [0.2580188810825348], "ldr": {"dL_sdil": -0.003795318305492401, "dL_bp": -0.003484100103378296, "ratio": 1.089325275646454}}, {"step": 1800, "epoch": 4, "train_loss": 0.015236591920256615, "cos_r_negg": [0.6065676212310791], "cos_innovation_negg": [0.6065676212310791], "cos_apical_negg": [0.6065676212310791], "cos_Ac_negg": [0.6065676212310791], "r_norm": [0.25817543268203735]}, {"step": 1850, "epoch": 4, "train_loss": 0.06860766559839249, "cos_r_negg": [0.6104742884635925], "cos_innovation_negg": [0.6104742884635925], "cos_apical_negg": [0.6104742884635925], "cos_Ac_negg": [0.6104742884635925], "r_norm": [0.26235654950141907]}, {"step": 1900, "epoch": 4, "train_loss": 0.01571567729115486, "cos_r_negg": [0.6265487670898438], "cos_innovation_negg": [0.6265487670898438], "cos_apical_negg": [0.6265487670898438], "cos_Ac_negg": [0.6265487670898438], "r_norm": [0.2330511510372162]}, {"step": 1950, "epoch": 4, "train_loss": 0.03187115862965584, "cos_r_negg": [0.6164643168449402], "cos_innovation_negg": [0.6164643168449402], "cos_apical_negg": [0.6164643168449402], "cos_Ac_negg": [0.6164643168449402], "r_norm": [0.22801163792610168]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9741, "test_loss": 0.09600425281524658}, {"step": 2000, "epoch": 5, "train_loss": 0.03756117448210716, "cos_r_negg": [0.6267037391662598], "cos_innovation_negg": [0.6267037391662598], "cos_apical_negg": [0.6267037391662598], "cos_Ac_negg": [0.6267037391662598], "r_norm": [0.2380337417125702], "ldr": {"dL_sdil": -0.0036298856139183044, "dL_bp": -0.0033759474754333496, "ratio": 1.0752198132029167}}, {"step": 2050, "epoch": 5, "train_loss": 0.01866176351904869, "cos_r_negg": [0.6198179721832275], "cos_innovation_negg": [0.6198179721832275], "cos_apical_negg": [0.6198179721832275], "cos_Ac_negg": [0.6198179721832275], "r_norm": [0.23081660270690918]}, {"step": 2100, "epoch": 5, "train_loss": 0.0061838384717702866, "cos_r_negg": [0.6299253702163696], "cos_innovation_negg": [0.6299253702163696], "cos_apical_negg": [0.6299253702163696], "cos_Ac_negg": [0.6299253702163696], "r_norm": [0.24187302589416504]}, {"step": 2150, "epoch": 5, "train_loss": 0.007370280101895332, "cos_r_negg": [0.6282036304473877], "cos_innovation_negg": [0.6282036304473877], "cos_apical_negg": [0.6282036304473877], "cos_Ac_negg": [0.6282036304473877], "r_norm": [0.22755968570709229]}, {"step": 2200, "epoch": 5, "train_loss": 0.02514728158712387, "cos_r_negg": [0.6256062984466553], "cos_innovation_negg": [0.6256062984466553], "cos_apical_negg": [0.6256062984466553], "cos_Ac_negg": [0.6256062984466553], "r_norm": [0.22950699925422668]}, {"step": 2250, "epoch": 5, "train_loss": 0.009106451645493507, "cos_r_negg": [0.62814861536026], "cos_innovation_negg": [0.62814861536026], "cos_apical_negg": [0.62814861536026], "cos_Ac_negg": [0.62814861536026], "r_norm": [0.24529218673706055], "ldr": {"dL_sdil": -0.004499912261962891, "dL_bp": -0.00419752299785614, "ratio": 1.0720399302782126}}, {"step": 2300, "epoch": 5, "train_loss": 0.012370266020298004, "cos_r_negg": [0.6376563310623169], "cos_innovation_negg": [0.6376563310623169], "cos_apical_negg": [0.6376563310623169], "cos_Ac_negg": [0.6376563310623169], "r_norm": [0.22183744609355927]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9726, "test_loss": 0.09546454582214356}, {"step": 2350, "epoch": 6, "train_loss": 0.013007815927267075, "cos_r_negg": [0.6363309621810913], "cos_innovation_negg": [0.6363309621810913], "cos_apical_negg": [0.6363309621810913], "cos_Ac_negg": [0.6363309621810913], "r_norm": [0.24499811232089996]}, {"step": 2400, "epoch": 6, "train_loss": 0.007484035566449165, "cos_r_negg": [0.644054651260376], "cos_innovation_negg": [0.644054651260376], "cos_apical_negg": [0.644054651260376], "cos_Ac_negg": [0.644054651260376], "r_norm": [0.24941711127758026]}, {"step": 2450, "epoch": 6, "train_loss": 0.015174789354205132, "cos_r_negg": [0.6380198001861572], "cos_innovation_negg": [0.6380198001861572], "cos_apical_negg": [0.6380198001861572], "cos_Ac_negg": [0.6380198001861572], "r_norm": [0.23315474390983582]}, {"step": 2500, "epoch": 6, "train_loss": 0.010220637544989586, "cos_r_negg": [0.637048602104187], "cos_innovation_negg": [0.637048602104187], "cos_apical_negg": [0.637048602104187], "cos_Ac_negg": [0.637048602104187], "r_norm": [0.22337706387043], "ldr": {"dL_sdil": -0.0035179927945137024, "dL_bp": -0.0032664239406585693, "ratio": 1.0770165962610512}}, {"step": 2550, "epoch": 6, "train_loss": 0.024868668988347054, "cos_r_negg": [0.6362561583518982], "cos_innovation_negg": [0.6362561583518982], "cos_apical_negg": [0.6362561583518982], "cos_Ac_negg": [0.6362561583518982], "r_norm": [0.20396193861961365]}, {"step": 2600, "epoch": 6, "train_loss": 0.0091217877343297, "cos_r_negg": [0.6308141946792603], "cos_innovation_negg": [0.6308141946792603], "cos_apical_negg": [0.6308141946792603], "cos_Ac_negg": [0.6308141946792603], "r_norm": [0.21701888740062714]}, {"step": 2650, "epoch": 6, "train_loss": 0.03402223810553551, "cos_r_negg": [0.6364970207214355], "cos_innovation_negg": [0.6364970207214355], "cos_apical_negg": [0.6364970207214355], "cos_Ac_negg": [0.6364970207214355], "r_norm": [0.21945659816265106]}, {"step": 2700, "epoch": 6, "train_loss": 0.0019882461056113243, "cos_r_negg": [0.6404767036437988], "cos_innovation_negg": [0.6404767036437988], "cos_apical_negg": [0.6404767036437988], "cos_Ac_negg": [0.6404767036437988], "r_norm": [0.24028022587299347]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9753, "test_loss": 0.09047592535018921}, {"step": 2750, "epoch": 7, "train_loss": 0.002221467439085245, "cos_r_negg": [0.6429628133773804], "cos_innovation_negg": [0.6429628133773804], "cos_apical_negg": [0.6429628133773804], "cos_Ac_negg": [0.6429628133773804], "r_norm": [0.21105587482452393], "ldr": {"dL_sdil": -0.003323715180158615, "dL_bp": -0.0030951760709285736, "ratio": 1.0738371918084384}}, {"step": 2800, "epoch": 7, "train_loss": 0.0029000183567404747, "cos_r_negg": [0.6497701406478882], "cos_innovation_negg": [0.6497701406478882], "cos_apical_negg": [0.6497701406478882], "cos_Ac_negg": [0.6497701406478882], "r_norm": [0.20301388204097748]}, {"step": 2850, "epoch": 7, "train_loss": 0.003696307074278593, "cos_r_negg": [0.6472421288490295], "cos_innovation_negg": [0.6472421288490295], "cos_apical_negg": [0.6472421288490295], "cos_Ac_negg": [0.6472421288490295], "r_norm": [0.19491742551326752]}, {"step": 2900, "epoch": 7, "train_loss": 0.00850644800812006, "cos_r_negg": [0.6430436372756958], "cos_innovation_negg": [0.6430436372756958], "cos_apical_negg": [0.6430436372756958], "cos_Ac_negg": [0.6430436372756958], "r_norm": [0.21279901266098022]}, {"step": 2950, "epoch": 7, "train_loss": 0.00397614948451519, "cos_r_negg": [0.6442052125930786], "cos_innovation_negg": [0.6442052125930786], "cos_apical_negg": [0.6442052125930786], "cos_Ac_negg": [0.6442052125930786], "r_norm": [0.19129197299480438]}, {"step": 3000, "epoch": 7, "train_loss": 0.008584652096033096, "cos_r_negg": [0.6444920301437378], "cos_innovation_negg": [0.6444920301437378], "cos_apical_negg": [0.6444920301437378], "cos_Ac_negg": [0.6444920301437378], "r_norm": [0.2191656231880188], "ldr": {"dL_sdil": -0.004183344542980194, "dL_bp": -0.003983132541179657, "ratio": 1.0502649609900356}}, {"step": 3050, "epoch": 7, "train_loss": 0.007573279086500406, "cos_r_negg": [0.6499322652816772], "cos_innovation_negg": [0.6499322652816772], "cos_apical_negg": [0.6499322652816772], "cos_Ac_negg": [0.6499322652816772], "r_norm": [0.20484137535095215]}, {"step": 3100, "epoch": 7, "train_loss": 0.0028750726487487555, "cos_r_negg": [0.6508527398109436], "cos_innovation_negg": [0.6508527398109436], "cos_apical_negg": [0.6508527398109436], "cos_Ac_negg": [0.6508527398109436], "r_norm": [0.2027432769536972]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9755, "test_loss": 0.09138434448242187}, {"step": 3150, "epoch": 8, "train_loss": 0.0042971959337592125, "cos_r_negg": [0.6481715440750122], "cos_innovation_negg": [0.6481715440750122], "cos_apical_negg": [0.6481715440750122], "cos_Ac_negg": [0.6481715440750122], "r_norm": [0.20355963706970215]}, {"step": 3200, "epoch": 8, "train_loss": 0.005786560475826263, "cos_r_negg": [0.6490699052810669], "cos_innovation_negg": [0.6490699052810669], "cos_apical_negg": [0.6490699052810669], "cos_Ac_negg": [0.6490699052810669], "r_norm": [0.2133893370628357]}, {"step": 3250, "epoch": 8, "train_loss": 0.0030147521756589413, "cos_r_negg": [0.6513056755065918], "cos_innovation_negg": [0.6513056755065918], "cos_apical_negg": [0.6513056755065918], "cos_Ac_negg": [0.6513056755065918], "r_norm": [0.21175767481327057], "ldr": {"dL_sdil": -0.00363057479262352, "dL_bp": -0.003551241010427475, "ratio": 1.0223397347471201}}, {"step": 3300, "epoch": 8, "train_loss": 0.003405182622373104, "cos_r_negg": [0.6518109440803528], "cos_innovation_negg": [0.6518109440803528], "cos_apical_negg": [0.6518109440803528], "cos_Ac_negg": [0.6518109440803528], "r_norm": [0.1912657916545868]}, {"step": 3350, "epoch": 8, "train_loss": 0.009097166359424591, "cos_r_negg": [0.6529719233512878], "cos_innovation_negg": [0.6529719233512878], "cos_apical_negg": [0.6529719233512878], "cos_Ac_negg": [0.6529719233512878], "r_norm": [0.2020569145679474]}, {"step": 3400, "epoch": 8, "train_loss": 0.003311256878077984, "cos_r_negg": [0.6481161713600159], "cos_innovation_negg": [0.6481161713600159], "cos_apical_negg": [0.6481161713600159], "cos_Ac_negg": [0.6481161713600159], "r_norm": [0.21467998623847961]}, {"step": 3450, "epoch": 8, "train_loss": 0.007715314626693726, "cos_r_negg": [0.651432991027832], "cos_innovation_negg": [0.651432991027832], "cos_apical_negg": [0.651432991027832], "cos_Ac_negg": [0.651432991027832], "r_norm": [0.21256482601165771]}, {"step": 3500, "epoch": 8, "train_loss": 0.007830852642655373, "cos_r_negg": [0.650222897529602], "cos_innovation_negg": [0.650222897529602], "cos_apical_negg": [0.650222897529602], "cos_Ac_negg": [0.650222897529602], "r_norm": [0.2102314829826355], "ldr": {"dL_sdil": -0.0033510178327560425, "dL_bp": -0.0031962506473064423, "ratio": 1.0484214795795275}}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9744, "test_loss": 0.09116042079925538}, {"step": 3550, "epoch": 9, "train_loss": 0.0037796073593199253, "cos_r_negg": [0.6507365703582764], "cos_innovation_negg": [0.6507365703582764], "cos_apical_negg": [0.6507365703582764], "cos_Ac_negg": [0.6507365703582764], "r_norm": [0.2036305069923401]}, {"step": 3600, "epoch": 9, "train_loss": 0.0031020422466099262, "cos_r_negg": [0.6536962985992432], "cos_innovation_negg": [0.6536962985992432], "cos_apical_negg": [0.6536962985992432], "cos_Ac_negg": [0.6536962985992432], "r_norm": [0.20583361387252808]}, {"step": 3650, "epoch": 9, "train_loss": 0.0031216482166200876, "cos_r_negg": [0.6543456315994263], "cos_innovation_negg": [0.6543456315994263], "cos_apical_negg": [0.6543456315994263], "cos_Ac_negg": [0.6543456315994263], "r_norm": [0.1966181993484497]}, {"step": 3700, "epoch": 9, "train_loss": 0.009208930656313896, "cos_r_negg": [0.6520483493804932], "cos_innovation_negg": [0.6520483493804932], "cos_apical_negg": [0.6520483493804932], "cos_Ac_negg": [0.6520483493804932], "r_norm": [0.20262029767036438]}, {"step": 3750, "epoch": 9, "train_loss": 0.0059758638963103294, "cos_r_negg": [0.6556220054626465], "cos_innovation_negg": [0.6556220054626465], "cos_apical_negg": [0.6556220054626465], "cos_Ac_negg": [0.6556220054626465], "r_norm": [0.19358861446380615], "ldr": {"dL_sdil": -0.0029736384749412537, "dL_bp": -0.0028409771621227264, "ratio": 1.0466956632341968}}, {"step": 3800, "epoch": 9, "train_loss": 0.0029000602662563324, "cos_r_negg": [0.653589129447937], "cos_innovation_negg": [0.653589129447937], "cos_apical_negg": [0.653589129447937], "cos_Ac_negg": [0.653589129447937], "r_norm": [0.19507010281085968]}, {"step": 3850, "epoch": 9, "train_loss": 0.005691350903362036, "cos_r_negg": [0.6553759574890137], "cos_innovation_negg": [0.6553759574890137], "cos_apical_negg": [0.6553759574890137], "cos_Ac_negg": [0.6553759574890137], "r_norm": [0.19933855533599854]}, {"step": 3900, "epoch": 9, "train_loss": 0.0045257010497152805, "cos_r_negg": [0.652214527130127], "cos_innovation_negg": [0.652214527130127], "cos_apical_negg": [0.652214527130127], "cos_Ac_negg": [0.652214527130127], "r_norm": [0.19489699602127075]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9759, "test_loss": 0.09030940465927124}, {"step": 3950, "epoch": 10, "train_loss": 0.0026249850634485483, "cos_r_negg": [0.6528564691543579], "cos_innovation_negg": [0.6528564691543579], "cos_apical_negg": [0.6528564691543579], "cos_Ac_negg": [0.6528564691543579], "r_norm": [0.19804993271827698]}, {"step": 4000, "epoch": 10, "train_loss": 0.0029309983365237713, "cos_r_negg": [0.6534816026687622], "cos_innovation_negg": [0.6534816026687622], "cos_apical_negg": [0.6534816026687622], "cos_Ac_negg": [0.6534816026687622], "r_norm": [0.20798970758914948], "ldr": {"dL_sdil": -0.0038084909319877625, "dL_bp": -0.0037738457322120667, "ratio": 1.0091803434040714}}, {"step": 4050, "epoch": 10, "train_loss": 0.0018851226195693016, "cos_r_negg": [0.6527501344680786], "cos_innovation_negg": [0.6527501344680786], "cos_apical_negg": [0.6527501344680786], "cos_Ac_negg": [0.6527501344680786], "r_norm": [0.19688081741333008]}, {"step": 4100, "epoch": 10, "train_loss": 0.0017330117989331484, "cos_r_negg": [0.655950129032135], "cos_innovation_negg": [0.655950129032135], "cos_apical_negg": [0.655950129032135], "cos_Ac_negg": [0.655950129032135], "r_norm": [0.18591895699501038]}, {"step": 4150, "epoch": 10, "train_loss": 0.002700213808566332, "cos_r_negg": [0.6562772989273071], "cos_innovation_negg": [0.6562772989273071], "cos_apical_negg": [0.6562772989273071], "cos_Ac_negg": [0.6562772989273071], "r_norm": [0.18376734852790833]}, {"step": 4200, "epoch": 10, "train_loss": 0.0033973422832787037, "cos_r_negg": [0.6593805551528931], "cos_innovation_negg": [0.6593805551528931], "cos_apical_negg": [0.6593805551528931], "cos_Ac_negg": [0.6593805551528931], "r_norm": [0.19360439479351044]}, {"step": 4250, "epoch": 10, "train_loss": 0.004042861983180046, "cos_r_negg": [0.6598274111747742], "cos_innovation_negg": [0.6598274111747742], "cos_apical_negg": [0.6598274111747742], "cos_Ac_negg": [0.6598274111747742], "r_norm": [0.19572336971759796], "ldr": {"dL_sdil": -0.003421485424041748, "dL_bp": -0.0033751577138900757, "ratio": 1.0137260875131897}}, {"step": 4300, "epoch": 10, "train_loss": 0.002331506460905075, "cos_r_negg": [0.6602481007575989], "cos_innovation_negg": [0.6602481007575989], "cos_apical_negg": [0.6602481007575989], "cos_Ac_negg": [0.6602481007575989], "r_norm": [0.1966586709022522]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9749, "test_loss": 0.09221825504302979}, {"step": 4350, "epoch": 11, "train_loss": 0.0033397795632481575, "cos_r_negg": [0.6602182984352112], "cos_innovation_negg": [0.6602182984352112], "cos_apical_negg": [0.6602182984352112], "cos_Ac_negg": [0.6602182984352112], "r_norm": [0.1877913922071457]}, {"step": 4400, "epoch": 11, "train_loss": 0.0015925051411613822, "cos_r_negg": [0.6591354608535767], "cos_innovation_negg": [0.6591354608535767], "cos_apical_negg": [0.6591354608535767], "cos_Ac_negg": [0.6591354608535767], "r_norm": [0.1881619542837143]}, {"step": 4450, "epoch": 11, "train_loss": 0.004382600542157888, "cos_r_negg": [0.6608985066413879], "cos_innovation_negg": [0.6608985066413879], "cos_apical_negg": [0.6608985066413879], "cos_Ac_negg": [0.6608985066413879], "r_norm": [0.18625503778457642]}, {"step": 4500, "epoch": 11, "train_loss": 0.0006002584705129266, "cos_r_negg": [0.661924421787262], "cos_innovation_negg": [0.661924421787262], "cos_apical_negg": [0.661924421787262], "cos_Ac_negg": [0.661924421787262], "r_norm": [0.1972832977771759], "ldr": {"dL_sdil": -0.0036121904850006104, "dL_bp": -0.003521658480167389, "ratio": 1.0257072073692162}}, {"step": 4550, "epoch": 11, "train_loss": 0.00401381915435195, "cos_r_negg": [0.6607362031936646], "cos_innovation_negg": [0.6607362031936646], "cos_apical_negg": [0.6607362031936646], "cos_Ac_negg": [0.6607362031936646], "r_norm": [0.20152641832828522]}, {"step": 4600, "epoch": 11, "train_loss": 0.0009283600375056267, "cos_r_negg": [0.6618537902832031], "cos_innovation_negg": [0.6618537902832031], "cos_apical_negg": [0.6618537902832031], "cos_Ac_negg": [0.6618537902832031], "r_norm": [0.19183026254177094]}, {"step": 4650, "epoch": 11, "train_loss": 0.0029345224611461163, "cos_r_negg": [0.6638206839561462], "cos_innovation_negg": [0.6638206839561462], "cos_apical_negg": [0.6638206839561462], "cos_Ac_negg": [0.6638206839561462], "r_norm": [0.19343842566013336]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9759, "test_loss": 0.09351576318740845}, {"step": 4700, "epoch": 12, "train_loss": 0.002603549277409911, "cos_r_negg": [0.6616617441177368], "cos_innovation_negg": [0.6616617441177368], "cos_apical_negg": [0.6616617441177368], "cos_Ac_negg": [0.6616617441177368], "r_norm": [0.1916811466217041]}, {"step": 4750, "epoch": 12, "train_loss": 0.0026827799156308174, "cos_r_negg": [0.6623045206069946], "cos_innovation_negg": [0.6623045206069946], "cos_apical_negg": [0.6623045206069946], "cos_Ac_negg": [0.6623045206069946], "r_norm": [0.2044047862291336], "ldr": {"dL_sdil": -0.0038029253482818604, "dL_bp": -0.003714621067047119, "ratio": 1.0237720832464177}}, {"step": 4800, "epoch": 12, "train_loss": 0.0021218121983110905, "cos_r_negg": [0.6612240076065063], "cos_innovation_negg": [0.6612240076065063], "cos_apical_negg": [0.6612240076065063], "cos_Ac_negg": [0.6612240076065063], "r_norm": [0.19510497152805328]}, {"step": 4850, "epoch": 12, "train_loss": 0.002113607246428728, "cos_r_negg": [0.660447359085083], "cos_innovation_negg": [0.660447359085083], "cos_apical_negg": [0.660447359085083], "cos_Ac_negg": [0.660447359085083], "r_norm": [0.195541650056839]}, {"step": 4900, "epoch": 12, "train_loss": 0.002382972976192832, "cos_r_negg": [0.6600865721702576], "cos_innovation_negg": [0.6600865721702576], "cos_apical_negg": [0.6600865721702576], "cos_Ac_negg": [0.6600865721702576], "r_norm": [0.1937941461801529]}, {"step": 4950, "epoch": 12, "train_loss": 0.001916486187838018, "cos_r_negg": [0.6600661277770996], "cos_innovation_negg": [0.6600661277770996], "cos_apical_negg": [0.6600661277770996], "cos_Ac_negg": [0.6600661277770996], "r_norm": [0.18756003677845]}, {"step": 5000, "epoch": 12, "train_loss": 0.0020850608125329018, "cos_r_negg": [0.6638098955154419], "cos_innovation_negg": [0.6638098955154419], "cos_apical_negg": [0.6638098955154419], "cos_Ac_negg": [0.6638098955154419], "r_norm": [0.1862126588821411], "ldr": {"dL_sdil": -0.0032613463699817657, "dL_bp": -0.0031703822314739227, "ratio": 1.0286918522330835}}, {"step": 5050, "epoch": 12, "train_loss": 0.004484038800001144, "cos_r_negg": [0.6618653535842896], "cos_innovation_negg": [0.6618653535842896], "cos_apical_negg": [0.6618653535842896], "cos_Ac_negg": [0.6618653535842896], "r_norm": [0.18285450339317322]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9751, "test_loss": 0.09388919305801392}, {"step": 5100, "epoch": 13, "train_loss": 0.0013214492937549949, "cos_r_negg": [0.6631682515144348], "cos_innovation_negg": [0.6631682515144348], "cos_apical_negg": [0.6631682515144348], "cos_Ac_negg": [0.6631682515144348], "r_norm": [0.19008173048496246]}, {"step": 5150, "epoch": 13, "train_loss": 0.0036634751595556736, "cos_r_negg": [0.6653921604156494], "cos_innovation_negg": [0.6653921604156494], "cos_apical_negg": [0.6653921604156494], "cos_Ac_negg": [0.6653921604156494], "r_norm": [0.19485771656036377]}, {"step": 5200, "epoch": 13, "train_loss": 0.002043203916400671, "cos_r_negg": [0.6653889417648315], "cos_innovation_negg": [0.6653889417648315], "cos_apical_negg": [0.6653889417648315], "cos_Ac_negg": [0.6653889417648315], "r_norm": [0.19730126857757568]}, {"step": 5250, "epoch": 13, "train_loss": 0.0028813304379582405, "cos_r_negg": [0.6647080183029175], "cos_innovation_negg": [0.6647080183029175], "cos_apical_negg": [0.6647080183029175], "cos_Ac_negg": [0.6647080183029175], "r_norm": [0.1881343126296997], "ldr": {"dL_sdil": -0.003411427140235901, "dL_bp": -0.003385886549949646, "ratio": 1.0075432504632014}}, {"step": 5300, "epoch": 13, "train_loss": 0.002271120436489582, "cos_r_negg": [0.6653828620910645], "cos_innovation_negg": [0.6653828620910645], "cos_apical_negg": [0.6653828620910645], "cos_Ac_negg": [0.6653828620910645], "r_norm": [0.1846219003200531]}, {"step": 5350, "epoch": 13, "train_loss": 0.0018193877767771482, "cos_r_negg": [0.6663289070129395], "cos_innovation_negg": [0.6663289070129395], "cos_apical_negg": [0.6663289070129395], "cos_Ac_negg": [0.6663289070129395], "r_norm": [0.1984713226556778]}, {"step": 5400, "epoch": 13, "train_loss": 0.0013861696934327483, "cos_r_negg": [0.6650077104568481], "cos_innovation_negg": [0.6650077104568481], "cos_apical_negg": [0.6650077104568481], "cos_Ac_negg": [0.6650077104568481], "r_norm": [0.1913594752550125]}, {"step": 5450, "epoch": 13, "train_loss": 0.0013751232763752341, "cos_r_negg": [0.6647144556045532], "cos_innovation_negg": [0.6647144556045532], "cos_apical_negg": [0.6647144556045532], "cos_Ac_negg": [0.6647144556045532], "r_norm": [0.19128507375717163]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9745, "test_loss": 0.09398780784606933}, {"step": 5500, "epoch": 14, "train_loss": 0.0017084911232814193, "cos_r_negg": [0.6638588905334473], "cos_innovation_negg": [0.6638588905334473], "cos_apical_negg": [0.6638588905334473], "cos_Ac_negg": [0.6638588905334473], "r_norm": [0.1904911994934082], "ldr": {"dL_sdil": -0.00334717333316803, "dL_bp": -0.0033628642559051514, "ratio": 0.9953340600324356}}, {"step": 5550, "epoch": 14, "train_loss": 0.0009113751584663987, "cos_r_negg": [0.6642800569534302], "cos_innovation_negg": [0.6642800569534302], "cos_apical_negg": [0.6642800569534302], "cos_Ac_negg": [0.6642800569534302], "r_norm": [0.18901148438453674]}, {"step": 5600, "epoch": 14, "train_loss": 0.0021965312771499157, "cos_r_negg": [0.6658776998519897], "cos_innovation_negg": [0.6658776998519897], "cos_apical_negg": [0.6658776998519897], "cos_Ac_negg": [0.6658776998519897], "r_norm": [0.18931293487548828]}, {"step": 5650, "epoch": 14, "train_loss": 0.0015544922789558768, "cos_r_negg": [0.6661753058433533], "cos_innovation_negg": [0.6661753058433533], "cos_apical_negg": [0.6661753058433533], "cos_Ac_negg": [0.6661753058433533], "r_norm": [0.18958419561386108]}, {"step": 5700, "epoch": 14, "train_loss": 0.0008453339105471969, "cos_r_negg": [0.6668015718460083], "cos_innovation_negg": [0.6668015718460083], "cos_apical_negg": [0.6668015718460083], "cos_Ac_negg": [0.6668015718460083], "r_norm": [0.1864953488111496]}, {"step": 5750, "epoch": 14, "train_loss": 0.0019126622937619686, "cos_r_negg": [0.6670761108398438], "cos_innovation_negg": [0.6670761108398438], "cos_apical_negg": [0.6670761108398438], "cos_Ac_negg": [0.6670761108398438], "r_norm": [0.19459354877471924], "ldr": {"dL_sdil": -0.00353061780333519, "dL_bp": -0.003529161214828491, "ratio": 1.0004127293761982}}, {"step": 5800, "epoch": 14, "train_loss": 0.003363291034474969, "cos_r_negg": [0.66578209400177], "cos_innovation_negg": [0.66578209400177], "cos_apical_negg": [0.66578209400177], "cos_Ac_negg": [0.66578209400177], "r_norm": [0.19483456015586853]}, {"step": 5850, "epoch": 14, "train_loss": 0.0020012310706079006, "cos_r_negg": [0.6661700010299683], "cos_innovation_negg": [0.6661700010299683], "cos_apical_negg": [0.6661700010299683], "cos_Ac_negg": [0.6661700010299683], "r_norm": [0.19089791178703308]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9754, "test_loss": 0.09481854515075684}, {"step": 5900, "epoch": 15, "train_loss": 0.0016229574102908373, "cos_r_negg": [0.6690080165863037], "cos_innovation_negg": [0.6690080165863037], "cos_apical_negg": [0.6690080165863037], "cos_Ac_negg": [0.6690080165863037], "r_norm": [0.191050723195076]}, {"step": 5950, "epoch": 15, "train_loss": 0.0024464367888867855, "cos_r_negg": [0.6685110330581665], "cos_innovation_negg": [0.6685110330581665], "cos_apical_negg": [0.6685110330581665], "cos_Ac_negg": [0.6685110330581665], "r_norm": [0.1900370568037033]}, {"step": 6000, "epoch": 15, "train_loss": 0.0024188498500734568, "cos_r_negg": [0.6676064729690552], "cos_innovation_negg": [0.6676064729690552], "cos_apical_negg": [0.6676064729690552], "cos_Ac_negg": [0.6676064729690552], "r_norm": [0.19068726897239685], "ldr": {"dL_sdil": -0.0036346428096294403, "dL_bp": -0.0036443471908569336, "ratio": 0.9973371414085244}}, {"step": 6050, "epoch": 15, "train_loss": 0.006012256257236004, "cos_r_negg": [0.6667393445968628], "cos_innovation_negg": [0.6667393445968628], "cos_apical_negg": [0.6667393445968628], "cos_Ac_negg": [0.6667393445968628], "r_norm": [0.19100013375282288]}, {"step": 6100, "epoch": 15, "train_loss": 0.002374814124777913, "cos_r_negg": [0.666784405708313], "cos_innovation_negg": [0.666784405708313], "cos_apical_negg": [0.666784405708313], "cos_Ac_negg": [0.666784405708313], "r_norm": [0.18990306556224823]}, {"step": 6150, "epoch": 15, "train_loss": 0.0016208906890824437, "cos_r_negg": [0.6696083545684814], "cos_innovation_negg": [0.6696083545684814], "cos_apical_negg": [0.6696083545684814], "cos_Ac_negg": [0.6696083545684814], "r_norm": [0.1844078153371811]}, {"step": 6200, "epoch": 15, "train_loss": 0.0023129465989768505, "cos_r_negg": [0.6680353879928589], "cos_innovation_negg": [0.6680353879928589], "cos_apical_negg": [0.6680353879928589], "cos_Ac_negg": [0.6680353879928589], "r_norm": [0.19079351425170898]}, {"step": 6250, "epoch": 15, "train_loss": 0.0017792124999687076, "cos_r_negg": [0.6695718765258789], "cos_innovation_negg": [0.6695718765258789], "cos_apical_negg": [0.6695718765258789], "cos_Ac_negg": [0.6695718765258789], "r_norm": [0.1842634230852127], "ldr": {"dL_sdil": -0.0033244118094444275, "dL_bp": -0.0034234896302223206, "ratio": 0.971059406780952}}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9747, "test_loss": 0.09474012088775635}, {"step": 6300, "epoch": 16, "train_loss": 0.002665690379217267, "cos_r_negg": [0.6682738065719604], "cos_innovation_negg": [0.6682738065719604], "cos_apical_negg": [0.6682738065719604], "cos_Ac_negg": [0.6682738065719604], "r_norm": [0.19305860996246338]}, {"step": 6350, "epoch": 16, "train_loss": 0.0011258770246058702, "cos_r_negg": [0.6670449376106262], "cos_innovation_negg": [0.6670449376106262], "cos_apical_negg": [0.6670449376106262], "cos_Ac_negg": [0.6670449376106262], "r_norm": [0.18983466923236847]}, {"step": 6400, "epoch": 16, "train_loss": 0.0018962530884891748, "cos_r_negg": [0.6696497201919556], "cos_innovation_negg": [0.6696497201919556], "cos_apical_negg": [0.6696497201919556], "cos_Ac_negg": [0.6696497201919556], "r_norm": [0.18576331436634064]}, {"step": 6450, "epoch": 16, "train_loss": 0.00105446542147547, "cos_r_negg": [0.6673800945281982], "cos_innovation_negg": [0.6673800945281982], "cos_apical_negg": [0.6673800945281982], "cos_Ac_negg": [0.6673800945281982], "r_norm": [0.18845504522323608]}, {"step": 6500, "epoch": 16, "train_loss": 0.0012440644204616547, "cos_r_negg": [0.6695458292961121], "cos_innovation_negg": [0.6695458292961121], "cos_apical_negg": [0.6695458292961121], "cos_Ac_negg": [0.6695458292961121], "r_norm": [0.18449927866458893], "ldr": {"dL_sdil": -0.003462698310613632, "dL_bp": -0.0035398341715335846, "ratio": 0.978209188006529}}, {"step": 6550, "epoch": 16, "train_loss": 0.001862668781541288, "cos_r_negg": [0.669255256652832], "cos_innovation_negg": [0.669255256652832], "cos_apical_negg": [0.669255256652832], "cos_Ac_negg": [0.669255256652832], "r_norm": [0.18135738372802734]}, {"step": 6600, "epoch": 16, "train_loss": 0.0021927286870777607, "cos_r_negg": [0.6696456670761108], "cos_innovation_negg": [0.6696456670761108], "cos_apical_negg": [0.6696456670761108], "cos_Ac_negg": [0.6696456670761108], "r_norm": [0.18003563582897186]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9751, "test_loss": 0.09571137046813964}, {"step": 6650, "epoch": 17, "train_loss": 0.0013606774155050516, "cos_r_negg": [0.6702663898468018], "cos_innovation_negg": [0.6702663898468018], "cos_apical_negg": [0.6702663898468018], "cos_Ac_negg": [0.6702663898468018], "r_norm": [0.18251150846481323]}, {"step": 6700, "epoch": 17, "train_loss": 0.0006907079368829727, "cos_r_negg": [0.6694053411483765], "cos_innovation_negg": [0.6694053411483765], "cos_apical_negg": [0.6694053411483765], "cos_Ac_negg": [0.6694053411483765], "r_norm": [0.1911943405866623]}, {"step": 6750, "epoch": 17, "train_loss": 0.00274057243950665, "cos_r_negg": [0.6713135242462158], "cos_innovation_negg": [0.6713135242462158], "cos_apical_negg": [0.6713135242462158], "cos_Ac_negg": [0.6713135242462158], "r_norm": [0.1962074190378189], "ldr": {"dL_sdil": -0.0039053037762641907, "dL_bp": -0.00398804247379303, "ratio": 0.9792533058329878}}, {"step": 6800, "epoch": 17, "train_loss": 0.0019151433371007442, "cos_r_negg": [0.6706393957138062], "cos_innovation_negg": [0.6706393957138062], "cos_apical_negg": [0.6706393957138062], "cos_Ac_negg": [0.6706393957138062], "r_norm": [0.18520523607730865]}, {"step": 6850, "epoch": 17, "train_loss": 0.0009233404998667538, "cos_r_negg": [0.6704190373420715], "cos_innovation_negg": [0.6704190373420715], "cos_apical_negg": [0.6704190373420715], "cos_Ac_negg": [0.6704190373420715], "r_norm": [0.18444395065307617]}, {"step": 6900, "epoch": 17, "train_loss": 0.0029176585376262665, "cos_r_negg": [0.6704696416854858], "cos_innovation_negg": [0.6704696416854858], "cos_apical_negg": [0.6704696416854858], "cos_Ac_negg": [0.6704696416854858], "r_norm": [0.18098634481430054]}, {"step": 6950, "epoch": 17, "train_loss": 0.001727807568386197, "cos_r_negg": [0.6715726852416992], "cos_innovation_negg": [0.6715726852416992], "cos_apical_negg": [0.6715726852416992], "cos_Ac_negg": [0.6715726852416992], "r_norm": [0.18495336174964905]}, {"step": 7000, "epoch": 17, "train_loss": 0.0014556820970028639, "cos_r_negg": [0.6697149872779846], "cos_innovation_negg": [0.6697149872779846], "cos_apical_negg": [0.6697149872779846], "cos_Ac_negg": [0.6697149872779846], "r_norm": [0.18415790796279907], "ldr": {"dL_sdil": -0.0034560412168502808, "dL_bp": -0.003533400595188141, "ratio": 0.9781062530970279}}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9754, "test_loss": 0.09646430683135987}, {"step": 7050, "epoch": 18, "train_loss": 0.0006269080331549048, "cos_r_negg": [0.6708806157112122], "cos_innovation_negg": [0.6708806157112122], "cos_apical_negg": [0.6708806157112122], "cos_Ac_negg": [0.6708806157112122], "r_norm": [0.18623986840248108]}, {"step": 7100, "epoch": 18, "train_loss": 0.001892173197120428, "cos_r_negg": [0.6714605093002319], "cos_innovation_negg": [0.6714605093002319], "cos_apical_negg": [0.6714605093002319], "cos_Ac_negg": [0.6714605093002319], "r_norm": [0.186018705368042]}, {"step": 7150, "epoch": 18, "train_loss": 0.0007320588920265436, "cos_r_negg": [0.6725932359695435], "cos_innovation_negg": [0.6725932359695435], "cos_apical_negg": [0.6725932359695435], "cos_Ac_negg": [0.6725932359695435], "r_norm": [0.1854158639907837]}, {"step": 7200, "epoch": 18, "train_loss": 0.0012678533094003797, "cos_r_negg": [0.6722308993339539], "cos_innovation_negg": [0.6722308993339539], "cos_apical_negg": [0.6722308993339539], "cos_Ac_negg": [0.6722308993339539], "r_norm": [0.19010186195373535]}, {"step": 7250, "epoch": 18, "train_loss": 0.0015020370483398438, "cos_r_negg": [0.6717742681503296], "cos_innovation_negg": [0.6717742681503296], "cos_apical_negg": [0.6717742681503296], "cos_Ac_negg": [0.6717742681503296], "r_norm": [0.1833387017250061], "ldr": {"dL_sdil": -0.003467392176389694, "dL_bp": -0.0035951808094978333, "ratio": 0.9644555754273766}}, {"step": 7300, "epoch": 18, "train_loss": 0.0008987576584331691, "cos_r_negg": [0.6711430549621582], "cos_innovation_negg": [0.6711430549621582], "cos_apical_negg": [0.6711430549621582], "cos_Ac_negg": [0.6711430549621582], "r_norm": [0.18265849351882935]}, {"step": 7350, "epoch": 18, "train_loss": 0.002781771821901202, "cos_r_negg": [0.6727261543273926], "cos_innovation_negg": [0.6727261543273926], "cos_apical_negg": [0.6727261543273926], "cos_Ac_negg": [0.6727261543273926], "r_norm": [0.18443873524665833]}, {"step": 7400, "epoch": 18, "train_loss": 0.0009053190005943179, "cos_r_negg": [0.6719615459442139], "cos_innovation_negg": [0.6719615459442139], "cos_apical_negg": [0.6719615459442139], "cos_Ac_negg": [0.6719615459442139], "r_norm": [0.18120646476745605]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9749, "test_loss": 0.09657257232666015}, {"step": 7450, "epoch": 19, "train_loss": 0.0006974199204705656, "cos_r_negg": [0.6701024770736694], "cos_innovation_negg": [0.6701024770736694], "cos_apical_negg": [0.6701024770736694], "cos_Ac_negg": [0.6701024770736694], "r_norm": [0.1833806335926056]}, {"step": 7500, "epoch": 19, "train_loss": 0.0014223279431462288, "cos_r_negg": [0.6712658405303955], "cos_innovation_negg": [0.6712658405303955], "cos_apical_negg": [0.6712658405303955], "cos_Ac_negg": [0.6712658405303955], "r_norm": [0.1833333820104599], "ldr": {"dL_sdil": -0.0034934431314468384, "dL_bp": -0.003617234528064728, "ratio": 0.9657773374500769}}, {"step": 7550, "epoch": 19, "train_loss": 0.0008628963842056692, "cos_r_negg": [0.6709852814674377], "cos_innovation_negg": [0.6709852814674377], "cos_apical_negg": [0.6709852814674377], "cos_Ac_negg": [0.6709852814674377], "r_norm": [0.18381869792938232]}, {"step": 7600, "epoch": 19, "train_loss": 0.000933677947614342, "cos_r_negg": [0.6724147796630859], "cos_innovation_negg": [0.6724147796630859], "cos_apical_negg": [0.6724147796630859], "cos_Ac_negg": [0.6724147796630859], "r_norm": [0.18401436507701874]}, {"step": 7650, "epoch": 19, "train_loss": 0.00210533756762743, "cos_r_negg": [0.673020601272583], "cos_innovation_negg": [0.673020601272583], "cos_apical_negg": [0.673020601272583], "cos_Ac_negg": [0.673020601272583], "r_norm": [0.18722890317440033]}, {"step": 7700, "epoch": 19, "train_loss": 0.002980827819555998, "cos_r_negg": [0.6724777817726135], "cos_innovation_negg": [0.6724777817726135], "cos_apical_negg": [0.6724777817726135], "cos_Ac_negg": [0.6724777817726135], "r_norm": [0.18157531321048737]}, {"step": 7750, "epoch": 19, "train_loss": 0.0012222775258123875, "cos_r_negg": [0.6729544401168823], "cos_innovation_negg": [0.6729544401168823], "cos_apical_negg": [0.6729544401168823], "cos_Ac_negg": [0.6729544401168823], "r_norm": [0.1831112504005432], "ldr": {"dL_sdil": -0.0035767406225204468, "dL_bp": -0.003687754273414612, "ratio": 0.9698966789369688}}, {"step": 7800, "epoch": 19, "train_loss": 0.0016017984598875046, "cos_r_negg": [0.6720913648605347], "cos_innovation_negg": [0.6720913648605347], "cos_apical_negg": [0.6720913648605347], "cos_Ac_negg": [0.6720913648605347], "r_norm": [0.18769578635692596]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9755, "test_loss": 0.09751326007843017}, {"step": 7850, "epoch": 20, "train_loss": 0.0011640674201771617, "cos_r_negg": [0.6719192862510681], "cos_innovation_negg": [0.6719192862510681], "cos_apical_negg": [0.6719192862510681], "cos_Ac_negg": [0.6719192862510681], "r_norm": [0.18266019225120544]}, {"step": 7900, "epoch": 20, "train_loss": 0.0013945633545517921, "cos_r_negg": [0.6719590425491333], "cos_innovation_negg": [0.6719590425491333], "cos_apical_negg": [0.6719590425491333], "cos_Ac_negg": [0.6719590425491333], "r_norm": [0.18217089772224426]}, {"step": 7950, "epoch": 20, "train_loss": 0.0006484786863438785, "cos_r_negg": [0.6715990900993347], "cos_innovation_negg": [0.6715990900993347], "cos_apical_negg": [0.6715990900993347], "cos_Ac_negg": [0.6715990900993347], "r_norm": [0.18335913121700287]}, {"step": 8000, "epoch": 20, "train_loss": 0.001479176338762045, "cos_r_negg": [0.6717581748962402], "cos_innovation_negg": [0.6717581748962402], "cos_apical_negg": [0.6717581748962402], "cos_Ac_negg": [0.6717581748962402], "r_norm": [0.18382728099822998], "ldr": {"dL_sdil": -0.003524467349052429, "dL_bp": -0.003672834485769272, "ratio": 0.9596041865508221}}, {"step": 8050, "epoch": 20, "train_loss": 0.0019113423768430948, "cos_r_negg": [0.6717259883880615], "cos_innovation_negg": [0.6717259883880615], "cos_apical_negg": [0.6717259883880615], "cos_Ac_negg": [0.6717259883880615], "r_norm": [0.18564698100090027]}, {"step": 8100, "epoch": 20, "train_loss": 0.0012723634717985988, "cos_r_negg": [0.6721281409263611], "cos_innovation_negg": [0.6721281409263611], "cos_apical_negg": [0.6721281409263611], "cos_Ac_negg": [0.6721281409263611], "r_norm": [0.18536216020584106]}, {"step": 8150, "epoch": 20, "train_loss": 0.0010790829546749592, "cos_r_negg": [0.672011137008667], "cos_innovation_negg": [0.672011137008667], "cos_apical_negg": [0.672011137008667], "cos_Ac_negg": [0.672011137008667], "r_norm": [0.1842576265335083]}, {"step": 8200, "epoch": 20, "train_loss": 0.0006283182883635163, "cos_r_negg": [0.6731815338134766], "cos_innovation_negg": [0.6731815338134766], "cos_apical_negg": [0.6731815338134766], "cos_Ac_negg": [0.6731815338134766], "r_norm": [0.18199115991592407]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9757, "test_loss": 0.09753635034561157}, {"step": 8250, "epoch": 21, "train_loss": 0.0013110461877658963, "cos_r_negg": [0.6730632781982422], "cos_innovation_negg": [0.6730632781982422], "cos_apical_negg": [0.6730632781982422], "cos_Ac_negg": [0.6730632781982422], "r_norm": [0.18280793726444244], "ldr": {"dL_sdil": -0.003532819449901581, "dL_bp": -0.0036873072385787964, "ratio": 0.9581028163151493}}, {"step": 8300, "epoch": 21, "train_loss": 0.001471756724640727, "cos_r_negg": [0.6730612516403198], "cos_innovation_negg": [0.6730612516403198], "cos_apical_negg": [0.6730612516403198], "cos_Ac_negg": [0.6730612516403198], "r_norm": [0.18328247964382172]}, {"step": 8350, "epoch": 21, "train_loss": 0.0016282268334180117, "cos_r_negg": [0.6724833250045776], "cos_innovation_negg": [0.6724833250045776], "cos_apical_negg": [0.6724833250045776], "cos_Ac_negg": [0.6724833250045776], "r_norm": [0.1813454031944275]}, {"step": 8400, "epoch": 21, "train_loss": 0.0023770348634570837, "cos_r_negg": [0.6731361150741577], "cos_innovation_negg": [0.6731361150741577], "cos_apical_negg": [0.6731361150741577], "cos_Ac_negg": [0.6731361150741577], "r_norm": [0.18158972263336182]}, {"step": 8450, "epoch": 21, "train_loss": 0.0012572102714329958, "cos_r_negg": [0.6724441051483154], "cos_innovation_negg": [0.6724441051483154], "cos_apical_negg": [0.6724441051483154], "cos_Ac_negg": [0.6724441051483154], "r_norm": [0.18274566531181335]}, {"step": 8500, "epoch": 21, "train_loss": 0.0011531193740665913, "cos_r_negg": [0.674044132232666], "cos_innovation_negg": [0.674044132232666], "cos_apical_negg": [0.674044132232666], "cos_Ac_negg": [0.674044132232666], "r_norm": [0.18216098845005035], "ldr": {"dL_sdil": -0.003632955253124237, "dL_bp": -0.0038025788962841034, "ratio": 0.955392472375623}}, {"step": 8550, "epoch": 21, "train_loss": 0.0008688338566571474, "cos_r_negg": [0.6746211051940918], "cos_innovation_negg": [0.6746211051940918], "cos_apical_negg": [0.6746211051940918], "cos_Ac_negg": [0.6746211051940918], "r_norm": [0.1817246973514557]}, {"step": 8600, "epoch": 21, "train_loss": 0.0026477486826479435, "cos_r_negg": [0.6738837361335754], "cos_innovation_negg": [0.6738837361335754], "cos_apical_negg": [0.6738837361335754], "cos_Ac_negg": [0.6738837361335754], "r_norm": [0.18445751070976257]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9756, "test_loss": 0.09840392980575562}, {"step": 8650, "epoch": 22, "train_loss": 0.0008905905997380614, "cos_r_negg": [0.6731122136116028], "cos_innovation_negg": [0.6731122136116028], "cos_apical_negg": [0.6731122136116028], "cos_Ac_negg": [0.6731122136116028], "r_norm": [0.1810935139656067]}, {"step": 8700, "epoch": 22, "train_loss": 0.0009468243806622922, "cos_r_negg": [0.6750270128250122], "cos_innovation_negg": [0.6750270128250122], "cos_apical_negg": [0.6750270128250122], "cos_Ac_negg": [0.6750270128250122], "r_norm": [0.18291456997394562]}, {"step": 8750, "epoch": 22, "train_loss": 0.0010155602358281612, "cos_r_negg": [0.674488365650177], "cos_innovation_negg": [0.674488365650177], "cos_apical_negg": [0.674488365650177], "cos_Ac_negg": [0.674488365650177], "r_norm": [0.18301863968372345], "ldr": {"dL_sdil": -0.0036750957369804382, "dL_bp": -0.0038258731365203857, "ratio": 0.9605900681596884}}, {"step": 8800, "epoch": 22, "train_loss": 0.001295067253522575, "cos_r_negg": [0.6745160222053528], "cos_innovation_negg": [0.6745160222053528], "cos_apical_negg": [0.6745160222053528], "cos_Ac_negg": [0.6745160222053528], "r_norm": [0.18012116849422455]}, {"step": 8850, "epoch": 22, "train_loss": 0.0011515431106090546, "cos_r_negg": [0.6755900979042053], "cos_innovation_negg": [0.6755900979042053], "cos_apical_negg": [0.6755900979042053], "cos_Ac_negg": [0.6755900979042053], "r_norm": [0.17844225466251373]}, {"step": 8900, "epoch": 22, "train_loss": 0.001693360973149538, "cos_r_negg": [0.6747000217437744], "cos_innovation_negg": [0.6747000217437744], "cos_apical_negg": [0.6747000217437744], "cos_Ac_negg": [0.6747000217437744], "r_norm": [0.18162968754768372]}, {"step": 8950, "epoch": 22, "train_loss": 0.001061367685906589, "cos_r_negg": [0.6748725771903992], "cos_innovation_negg": [0.6748725771903992], "cos_apical_negg": [0.6748725771903992], "cos_Ac_negg": [0.6748725771903992], "r_norm": [0.18417471647262573]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9752, "test_loss": 0.09848454608917237}, {"step": 9000, "epoch": 23, "train_loss": 0.0005789534188807011, "cos_r_negg": [0.6747207045555115], "cos_innovation_negg": [0.6747207045555115], "cos_apical_negg": [0.6747207045555115], "cos_Ac_negg": [0.6747207045555115], "r_norm": [0.17871084809303284], "ldr": {"dL_sdil": -0.00350990891456604, "dL_bp": -0.00371580570936203, "ratio": 0.9445889233989738}}, {"step": 9050, "epoch": 23, "train_loss": 0.0008528418838977814, "cos_r_negg": [0.6748285293579102], "cos_innovation_negg": [0.6748285293579102], "cos_apical_negg": [0.6748285293579102], "cos_Ac_negg": [0.6748285293579102], "r_norm": [0.18366293609142303]}, {"step": 9100, "epoch": 23, "train_loss": 0.0009446109179407358, "cos_r_negg": [0.6741859316825867], "cos_innovation_negg": [0.6741859316825867], "cos_apical_negg": [0.6741859316825867], "cos_Ac_negg": [0.6741859316825867], "r_norm": [0.18150323629379272]}, {"step": 9150, "epoch": 23, "train_loss": 0.0015668785199522972, "cos_r_negg": [0.6739867925643921], "cos_innovation_negg": [0.6739867925643921], "cos_apical_negg": [0.6739867925643921], "cos_Ac_negg": [0.6739867925643921], "r_norm": [0.18237099051475525]}, {"step": 9200, "epoch": 23, "train_loss": 0.001606656238436699, "cos_r_negg": [0.674719512462616], "cos_innovation_negg": [0.674719512462616], "cos_apical_negg": [0.674719512462616], "cos_Ac_negg": [0.674719512462616], "r_norm": [0.18110883235931396]}, {"step": 9250, "epoch": 23, "train_loss": 0.001103358925320208, "cos_r_negg": [0.6748310327529907], "cos_innovation_negg": [0.6748310327529907], "cos_apical_negg": [0.6748310327529907], "cos_Ac_negg": [0.6748310327529907], "r_norm": [0.1805419921875], "ldr": {"dL_sdil": -0.0035963505506515503, "dL_bp": -0.0037932991981506348, "ratio": 0.9480798541820524}}, {"step": 9300, "epoch": 23, "train_loss": 0.0008418713696300983, "cos_r_negg": [0.6727344393730164], "cos_innovation_negg": [0.6727344393730164], "cos_apical_negg": [0.6727344393730164], "cos_Ac_negg": [0.6727344393730164], "r_norm": [0.18265530467033386]}, {"step": 9350, "epoch": 23, "train_loss": 0.0011481157271191478, "cos_r_negg": [0.6760683655738831], "cos_innovation_negg": [0.6760683655738831], "cos_apical_negg": [0.6760683655738831], "cos_Ac_negg": [0.6760683655738831], "r_norm": [0.18025949597358704]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9754, "test_loss": 0.09906525707244873}, {"step": 9400, "epoch": 24, "train_loss": 0.0008236198918893933, "cos_r_negg": [0.674741804599762], "cos_innovation_negg": [0.674741804599762], "cos_apical_negg": [0.674741804599762], "cos_Ac_negg": [0.674741804599762], "r_norm": [0.18068352341651917]}, {"step": 9450, "epoch": 24, "train_loss": 0.0006694122566841543, "cos_r_negg": [0.6754828691482544], "cos_innovation_negg": [0.6754828691482544], "cos_apical_negg": [0.6754828691482544], "cos_Ac_negg": [0.6754828691482544], "r_norm": [0.18071813881397247]}, {"step": 9500, "epoch": 24, "train_loss": 0.0011944910511374474, "cos_r_negg": [0.6765767335891724], "cos_innovation_negg": [0.6765767335891724], "cos_apical_negg": [0.6765767335891724], "cos_Ac_negg": [0.6765767335891724], "r_norm": [0.18075284361839294], "ldr": {"dL_sdil": -0.003710828721523285, "dL_bp": -0.003925025463104248, "ratio": 0.9454279357944451}}, {"step": 9550, "epoch": 24, "train_loss": 0.0011825578985735774, "cos_r_negg": [0.6743413209915161], "cos_innovation_negg": [0.6743413209915161], "cos_apical_negg": [0.6743413209915161], "cos_Ac_negg": [0.6743413209915161], "r_norm": [0.18242913484573364]}, {"step": 9600, "epoch": 24, "train_loss": 0.0007622168632224202, "cos_r_negg": [0.6747339367866516], "cos_innovation_negg": [0.6747339367866516], "cos_apical_negg": [0.6747339367866516], "cos_Ac_negg": [0.6747339367866516], "r_norm": [0.17969125509262085]}, {"step": 9650, "epoch": 24, "train_loss": 0.001686361851170659, "cos_r_negg": [0.6759655475616455], "cos_innovation_negg": [0.6759655475616455], "cos_apical_negg": [0.6759655475616455], "cos_Ac_negg": [0.6759655475616455], "r_norm": [0.1803051233291626]}, {"step": 9700, "epoch": 24, "train_loss": 0.0007466188399121165, "cos_r_negg": [0.6754062175750732], "cos_innovation_negg": [0.6754062175750732], "cos_apical_negg": [0.6754062175750732], "cos_Ac_negg": [0.6754062175750732], "r_norm": [0.17914724349975586]}, {"step": 9750, "epoch": 24, "train_loss": 0.00262606143951416, "cos_r_negg": [0.674490213394165], "cos_innovation_negg": [0.674490213394165], "cos_apical_negg": [0.674490213394165], "cos_Ac_negg": [0.674490213394165], "r_norm": [0.179164320230484], "ldr": {"dL_sdil": -0.0036454275250434875, "dL_bp": -0.0038645267486572266, "ratio": 0.9433050311555309}}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9753, "test_loss": 0.09880605974197387}], "final": {"test_acc": 0.9753, "test_loss": 0.09880605974197387, "wall_s": 15.69247055053711, "cos_r_negg": [0.6742957830429077], "cos_innovation_negg": [0.6742957830429077], "cos_apical_negg": [0.6742957830429077], "cos_Ac_negg": [0.6742957830429077]}} \ No newline at end of file diff --git a/results/ep_archmatch_v1_mnist_sdil_w500_d1_s1.json b/results/ep_archmatch_v1_mnist_sdil_w500_d1_s1.json new file mode 100644 index 0000000..d0175d6 --- /dev/null +++ b/results/ep_archmatch_v1_mnist_sdil_w500_d1_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 1, "width": 500, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_archmatch_v1_mnist_sdil_w500_d1_s1", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.5871329307556152, "cos_r_negg": [0.001100305700674653], "cos_innovation_negg": [0.001100305700674653], "cos_apical_negg": [0.001100305700674653], "cos_Ac_negg": [0.001100305700674653], "r_norm": [6.760903835296631], "ldr": {"dL_sdil": -0.1918637752532959, "dL_bp": -0.43957364559173584, "ratio": 0.43647697530869217}}, {"step": 50, "epoch": 0, "train_loss": 0.48861250281333923, "cos_r_negg": [0.43220454454421997], "cos_innovation_negg": [0.43220454454421997], "cos_apical_negg": [0.43220454454421997], "cos_Ac_negg": [0.43220454454421997], "r_norm": [1.3571507930755615]}, {"step": 100, "epoch": 0, "train_loss": 0.1364513337612152, "cos_r_negg": [0.4309086799621582], "cos_innovation_negg": [0.4309086799621582], "cos_apical_negg": [0.4309086799621582], "cos_Ac_negg": [0.4309086799621582], "r_norm": [1.0939793586730957]}, {"step": 150, "epoch": 0, "train_loss": 0.1887732446193695, "cos_r_negg": [0.4542691111564636], "cos_innovation_negg": [0.4542691111564636], "cos_apical_negg": [0.4542691111564636], "cos_Ac_negg": [0.4542691111564636], "r_norm": [0.8955298662185669]}, {"step": 200, "epoch": 0, "train_loss": 0.2780294418334961, "cos_r_negg": [0.4592840075492859], "cos_innovation_negg": [0.4592840075492859], "cos_apical_negg": [0.4592840075492859], "cos_Ac_negg": [0.4592840075492859], "r_norm": [0.8773269057273865]}, {"step": 250, "epoch": 0, "train_loss": 0.3455576002597809, "cos_r_negg": [0.462536484003067], "cos_innovation_negg": [0.462536484003067], "cos_apical_negg": [0.462536484003067], "cos_Ac_negg": [0.462536484003067], "r_norm": [0.856778085231781], "ldr": {"dL_sdil": -0.02157461643218994, "dL_bp": -0.019618839025497437, "ratio": 1.099688743261041}}, {"step": 300, "epoch": 0, "train_loss": 0.13695314526557922, "cos_r_negg": [0.4832298755645752], "cos_innovation_negg": [0.4832298755645752], "cos_apical_negg": [0.4832298755645752], "cos_Ac_negg": [0.4832298755645752], "r_norm": [0.7859394550323486]}, {"step": 350, "epoch": 0, "train_loss": 0.22819146513938904, "cos_r_negg": [0.4699955880641937], "cos_innovation_negg": [0.4699955880641937], "cos_apical_negg": [0.4699955880641937], "cos_Ac_negg": [0.4699955880641937], "r_norm": [0.6377558708190918]}, {"epoch_end": 0, "step": 391, "test_acc": 0.9503, "test_loss": 0.16497855377197265}, {"step": 400, "epoch": 1, "train_loss": 0.13905087113380432, "cos_r_negg": [0.48866862058639526], "cos_innovation_negg": [0.48866862058639526], "cos_apical_negg": [0.48866862058639526], "cos_Ac_negg": [0.48866862058639526], "r_norm": [0.6100770235061646]}, {"step": 450, "epoch": 1, "train_loss": 0.1279246211051941, "cos_r_negg": [0.5000467896461487], "cos_innovation_negg": [0.5000467896461487], "cos_apical_negg": [0.5000467896461487], "cos_Ac_negg": [0.5000467896461487], "r_norm": [0.5184377431869507]}, {"step": 500, "epoch": 1, "train_loss": 0.17944064736366272, "cos_r_negg": [0.5100876688957214], "cos_innovation_negg": [0.5100876688957214], "cos_apical_negg": [0.5100876688957214], "cos_Ac_negg": [0.5100876688957214], "r_norm": [0.43126726150512695], "ldr": {"dL_sdil": -0.004412665963172913, "dL_bp": -0.003992907702922821, "ratio": 1.1051259611993602}}, {"step": 550, "epoch": 1, "train_loss": 0.06834127753973007, "cos_r_negg": [0.5085099339485168], "cos_innovation_negg": [0.5085099339485168], "cos_apical_negg": [0.5085099339485168], "cos_Ac_negg": [0.5085099339485168], "r_norm": [0.5281723737716675]}, {"step": 600, "epoch": 1, "train_loss": 0.058914653956890106, "cos_r_negg": [0.5260806679725647], "cos_innovation_negg": [0.5260806679725647], "cos_apical_negg": [0.5260806679725647], "cos_Ac_negg": [0.5260806679725647], "r_norm": [0.502312421798706]}, {"step": 650, "epoch": 1, "train_loss": 0.06621991097927094, "cos_r_negg": [0.5217529535293579], "cos_innovation_negg": [0.5217529535293579], "cos_apical_negg": [0.5217529535293579], "cos_Ac_negg": [0.5217529535293579], "r_norm": [0.47615846991539]}, {"step": 700, "epoch": 1, "train_loss": 0.10719330608844757, "cos_r_negg": [0.5240798592567444], "cos_innovation_negg": [0.5240798592567444], "cos_apical_negg": [0.5240798592567444], "cos_Ac_negg": [0.5240798592567444], "r_norm": [0.42596250772476196]}, {"step": 750, "epoch": 1, "train_loss": 0.16641952097415924, "cos_r_negg": [0.5301454663276672], "cos_innovation_negg": [0.5301454663276672], "cos_apical_negg": [0.5301454663276672], "cos_Ac_negg": [0.5301454663276672], "r_norm": [0.49993330240249634], "ldr": {"dL_sdil": -0.012946158647537231, "dL_bp": -0.011827364563941956, "ratio": 1.0945936922419843}}, {"epoch_end": 1, "step": 782, "test_acc": 0.9564, "test_loss": 0.13900784950256348}, {"step": 800, "epoch": 2, "train_loss": 0.10563233494758606, "cos_r_negg": [0.530672550201416], "cos_innovation_negg": [0.530672550201416], "cos_apical_negg": [0.530672550201416], "cos_Ac_negg": [0.530672550201416], "r_norm": [0.45425549149513245]}, {"step": 850, "epoch": 2, "train_loss": 0.039473116397857666, "cos_r_negg": [0.5469871759414673], "cos_innovation_negg": [0.5469871759414673], "cos_apical_negg": [0.5469871759414673], "cos_Ac_negg": [0.5469871759414673], "r_norm": [0.3899500370025635]}, {"step": 900, "epoch": 2, "train_loss": 0.05765703693032265, "cos_r_negg": [0.553665041923523], "cos_innovation_negg": [0.553665041923523], "cos_apical_negg": [0.553665041923523], "cos_Ac_negg": [0.553665041923523], "r_norm": [0.46398618817329407]}, {"step": 950, "epoch": 2, "train_loss": 0.11174166947603226, "cos_r_negg": [0.5645879507064819], "cos_innovation_negg": [0.5645879507064819], "cos_apical_negg": [0.5645879507064819], "cos_Ac_negg": [0.5645879507064819], "r_norm": [0.32989072799682617]}, {"step": 1000, "epoch": 2, "train_loss": 0.0692964717745781, "cos_r_negg": [0.5555451512336731], "cos_innovation_negg": [0.5555451512336731], "cos_apical_negg": [0.5555451512336731], "cos_Ac_negg": [0.5555451512336731], "r_norm": [0.3335726857185364], "ldr": {"dL_sdil": -0.0040866658091545105, "dL_bp": -0.0035779550671577454, "ratio": 1.1421791868395024}}, {"step": 1050, "epoch": 2, "train_loss": 0.04302312433719635, "cos_r_negg": [0.5554473400115967], "cos_innovation_negg": [0.5554473400115967], "cos_apical_negg": [0.5554473400115967], "cos_Ac_negg": [0.5554473400115967], "r_norm": [0.39047497510910034]}, {"step": 1100, "epoch": 2, "train_loss": 0.09594385325908661, "cos_r_negg": [0.5579342842102051], "cos_innovation_negg": [0.5579342842102051], "cos_apical_negg": [0.5579342842102051], "cos_Ac_negg": [0.5579342842102051], "r_norm": [0.34934842586517334]}, {"step": 1150, "epoch": 2, "train_loss": 0.09900315850973129, "cos_r_negg": [0.5719032287597656], "cos_innovation_negg": [0.5719032287597656], "cos_apical_negg": [0.5719032287597656], "cos_Ac_negg": [0.5719032287597656], "r_norm": [0.3224126994609833]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9682, "test_loss": 0.1040596591949463}, {"step": 1200, "epoch": 3, "train_loss": 0.05156555026769638, "cos_r_negg": [0.5610235333442688], "cos_innovation_negg": [0.5610235333442688], "cos_apical_negg": [0.5610235333442688], "cos_Ac_negg": [0.5610235333442688], "r_norm": [0.4339083135128021]}, {"step": 1250, "epoch": 3, "train_loss": 0.10830618441104889, "cos_r_negg": [0.5736144781112671], "cos_innovation_negg": [0.5736144781112671], "cos_apical_negg": [0.5736144781112671], "cos_Ac_negg": [0.5736144781112671], "r_norm": [0.34213781356811523], "ldr": {"dL_sdil": -0.00485578179359436, "dL_bp": -0.004557289183139801, "ratio": 1.0654978427875208}}, {"step": 1300, "epoch": 3, "train_loss": 0.01793331280350685, "cos_r_negg": [0.5749954581260681], "cos_innovation_negg": [0.5749954581260681], "cos_apical_negg": [0.5749954581260681], "cos_Ac_negg": [0.5749954581260681], "r_norm": [0.397167831659317]}, {"step": 1350, "epoch": 3, "train_loss": 0.032390326261520386, "cos_r_negg": [0.5779156684875488], "cos_innovation_negg": [0.5779156684875488], "cos_apical_negg": [0.5779156684875488], "cos_Ac_negg": [0.5779156684875488], "r_norm": [0.30049705505371094]}, {"step": 1400, "epoch": 3, "train_loss": 0.04874143376946449, "cos_r_negg": [0.5797494053840637], "cos_innovation_negg": [0.5797494053840637], "cos_apical_negg": [0.5797494053840637], "cos_Ac_negg": [0.5797494053840637], "r_norm": [0.33478492498397827]}, {"step": 1450, "epoch": 3, "train_loss": 0.018085453659296036, "cos_r_negg": [0.5899109840393066], "cos_innovation_negg": [0.5899109840393066], "cos_apical_negg": [0.5899109840393066], "cos_Ac_negg": [0.5899109840393066], "r_norm": [0.3004092872142792]}, {"step": 1500, "epoch": 3, "train_loss": 0.0623151957988739, "cos_r_negg": [0.589335560798645], "cos_innovation_negg": [0.589335560798645], "cos_apical_negg": [0.589335560798645], "cos_Ac_negg": [0.589335560798645], "r_norm": [0.26107895374298096], "ldr": {"dL_sdil": -0.0031843073666095734, "dL_bp": -0.0029101520776748657, "ratio": 1.094206516229128}}, {"step": 1550, "epoch": 3, "train_loss": 0.0387909933924675, "cos_r_negg": [0.5903338193893433], "cos_innovation_negg": [0.5903338193893433], "cos_apical_negg": [0.5903338193893433], "cos_Ac_negg": [0.5903338193893433], "r_norm": [0.2533503770828247]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9695, "test_loss": 0.1060980297088623}, {"step": 1600, "epoch": 4, "train_loss": 0.04044149070978165, "cos_r_negg": [0.5882481336593628], "cos_innovation_negg": [0.5882481336593628], "cos_apical_negg": [0.5882481336593628], "cos_Ac_negg": [0.5882481336593628], "r_norm": [0.24958622455596924]}, {"step": 1650, "epoch": 4, "train_loss": 0.03958461061120033, "cos_r_negg": [0.5960929989814758], "cos_innovation_negg": [0.5960929989814758], "cos_apical_negg": [0.5960929989814758], "cos_Ac_negg": [0.5960929989814758], "r_norm": [0.2710718512535095]}, {"step": 1700, "epoch": 4, "train_loss": 0.03976733982563019, "cos_r_negg": [0.6068387627601624], "cos_innovation_negg": [0.6068387627601624], "cos_apical_negg": [0.6068387627601624], "cos_Ac_negg": [0.6068387627601624], "r_norm": [0.25744855403900146]}, {"step": 1750, "epoch": 4, "train_loss": 0.04680148512125015, "cos_r_negg": [0.606350302696228], "cos_innovation_negg": [0.606350302696228], "cos_apical_negg": [0.606350302696228], "cos_Ac_negg": [0.606350302696228], "r_norm": [0.2666856050491333], "ldr": {"dL_sdil": -0.0035785138607025146, "dL_bp": -0.003533497452735901, "ratio": 1.0127399010665081}}, {"step": 1800, "epoch": 4, "train_loss": 0.045200884342193604, "cos_r_negg": [0.6007728576660156], "cos_innovation_negg": [0.6007728576660156], "cos_apical_negg": [0.6007728576660156], "cos_Ac_negg": [0.6007728576660156], "r_norm": [0.27807140350341797]}, {"step": 1850, "epoch": 4, "train_loss": 0.09985603392124176, "cos_r_negg": [0.6068681478500366], "cos_innovation_negg": [0.6068681478500366], "cos_apical_negg": [0.6068681478500366], "cos_Ac_negg": [0.6068681478500366], "r_norm": [0.2567717730998993]}, {"step": 1900, "epoch": 4, "train_loss": 0.008486926555633545, "cos_r_negg": [0.612114429473877], "cos_innovation_negg": [0.612114429473877], "cos_apical_negg": [0.612114429473877], "cos_Ac_negg": [0.612114429473877], "r_norm": [0.2662491798400879]}, {"step": 1950, "epoch": 4, "train_loss": 0.07286867499351501, "cos_r_negg": [0.6119774580001831], "cos_innovation_negg": [0.6119774580001831], "cos_apical_negg": [0.6119774580001831], "cos_Ac_negg": [0.6119774580001831], "r_norm": [0.2504240572452545]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9727, "test_loss": 0.093028804397583}, {"step": 2000, "epoch": 5, "train_loss": 0.02292187511920929, "cos_r_negg": [0.6196531057357788], "cos_innovation_negg": [0.6196531057357788], "cos_apical_negg": [0.6196531057357788], "cos_Ac_negg": [0.6196531057357788], "r_norm": [0.2947973608970642], "ldr": {"dL_sdil": -0.004512958228588104, "dL_bp": -0.0045277029275894165, "ratio": 0.996743448225928}}, {"step": 2050, "epoch": 5, "train_loss": 0.008723124861717224, "cos_r_negg": [0.6156461834907532], "cos_innovation_negg": [0.6156461834907532], "cos_apical_negg": [0.6156461834907532], "cos_Ac_negg": [0.6156461834907532], "r_norm": [0.2753215730190277]}, {"step": 2100, "epoch": 5, "train_loss": 0.015341127291321754, "cos_r_negg": [0.6275525093078613], "cos_innovation_negg": [0.6275525093078613], "cos_apical_negg": [0.6275525093078613], "cos_Ac_negg": [0.6275525093078613], "r_norm": [0.2597804069519043]}, {"step": 2150, "epoch": 5, "train_loss": 0.0046372502110898495, "cos_r_negg": [0.6227806806564331], "cos_innovation_negg": [0.6227806806564331], "cos_apical_negg": [0.6227806806564331], "cos_Ac_negg": [0.6227806806564331], "r_norm": [0.24430224299430847]}, {"step": 2200, "epoch": 5, "train_loss": 0.027534987777471542, "cos_r_negg": [0.617467999458313], "cos_innovation_negg": [0.617467999458313], "cos_apical_negg": [0.617467999458313], "cos_Ac_negg": [0.617467999458313], "r_norm": [0.24936538934707642]}, {"step": 2250, "epoch": 5, "train_loss": 0.006457236595451832, "cos_r_negg": [0.6247696876525879], "cos_innovation_negg": [0.6247696876525879], "cos_apical_negg": [0.6247696876525879], "cos_Ac_negg": [0.6247696876525879], "r_norm": [0.23975923657417297], "ldr": {"dL_sdil": -0.0039103031158447266, "dL_bp": -0.003700532019138336, "ratio": 1.0566867400745354}}, {"step": 2300, "epoch": 5, "train_loss": 0.01453243289142847, "cos_r_negg": [0.6218576431274414], "cos_innovation_negg": [0.6218576431274414], "cos_apical_negg": [0.6218576431274414], "cos_Ac_negg": [0.6218576431274414], "r_norm": [0.23658186197280884]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9738, "test_loss": 0.09222160034179687}, {"step": 2350, "epoch": 6, "train_loss": 0.008035858161747456, "cos_r_negg": [0.6246671676635742], "cos_innovation_negg": [0.6246671676635742], "cos_apical_negg": [0.6246671676635742], "cos_Ac_negg": [0.6246671676635742], "r_norm": [0.21404096484184265]}, {"step": 2400, "epoch": 6, "train_loss": 0.009463842026889324, "cos_r_negg": [0.6295669674873352], "cos_innovation_negg": [0.6295669674873352], "cos_apical_negg": [0.6295669674873352], "cos_Ac_negg": [0.6295669674873352], "r_norm": [0.2530636191368103]}, {"step": 2450, "epoch": 6, "train_loss": 0.019154634326696396, "cos_r_negg": [0.6314516663551331], "cos_innovation_negg": [0.6314516663551331], "cos_apical_negg": [0.6314516663551331], "cos_Ac_negg": [0.6314516663551331], "r_norm": [0.25009602308273315]}, {"step": 2500, "epoch": 6, "train_loss": 0.010175041854381561, "cos_r_negg": [0.6321192979812622], "cos_innovation_negg": [0.6321192979812622], "cos_apical_negg": [0.6321192979812622], "cos_Ac_negg": [0.6321192979812622], "r_norm": [0.2250152975320816], "ldr": {"dL_sdil": -0.003260582685470581, "dL_bp": -0.0030800029635429382, "ratio": 1.0586297234337467}}, {"step": 2550, "epoch": 6, "train_loss": 0.009414475411176682, "cos_r_negg": [0.6312077045440674], "cos_innovation_negg": [0.6312077045440674], "cos_apical_negg": [0.6312077045440674], "cos_Ac_negg": [0.6312077045440674], "r_norm": [0.21666473150253296]}, {"step": 2600, "epoch": 6, "train_loss": 0.013411739841103554, "cos_r_negg": [0.6316648721694946], "cos_innovation_negg": [0.6316648721694946], "cos_apical_negg": [0.6316648721694946], "cos_Ac_negg": [0.6316648721694946], "r_norm": [0.23372584581375122]}, {"step": 2650, "epoch": 6, "train_loss": 0.026791274547576904, "cos_r_negg": [0.6384471654891968], "cos_innovation_negg": [0.6384471654891968], "cos_apical_negg": [0.6384471654891968], "cos_Ac_negg": [0.6384471654891968], "r_norm": [0.23166270554065704]}, {"step": 2700, "epoch": 6, "train_loss": 0.00908804964274168, "cos_r_negg": [0.6370428204536438], "cos_innovation_negg": [0.6370428204536438], "cos_apical_negg": [0.6370428204536438], "cos_Ac_negg": [0.6370428204536438], "r_norm": [0.24408280849456787]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9745, "test_loss": 0.08660848941802979}, {"step": 2750, "epoch": 7, "train_loss": 0.001891186460852623, "cos_r_negg": [0.6397808194160461], "cos_innovation_negg": [0.6397808194160461], "cos_apical_negg": [0.6397808194160461], "cos_Ac_negg": [0.6397808194160461], "r_norm": [0.20459353923797607], "ldr": {"dL_sdil": -0.002634003758430481, "dL_bp": -0.002552248537540436, "ratio": 1.0320326252273344}}, {"step": 2800, "epoch": 7, "train_loss": 0.004815737716853619, "cos_r_negg": [0.6417429447174072], "cos_innovation_negg": [0.6417429447174072], "cos_apical_negg": [0.6417429447174072], "cos_Ac_negg": [0.6417429447174072], "r_norm": [0.19778858125209808]}, {"step": 2850, "epoch": 7, "train_loss": 0.005631942767649889, "cos_r_negg": [0.6427261829376221], "cos_innovation_negg": [0.6427261829376221], "cos_apical_negg": [0.6427261829376221], "cos_Ac_negg": [0.6427261829376221], "r_norm": [0.2211076021194458]}, {"step": 2900, "epoch": 7, "train_loss": 0.006040941923856735, "cos_r_negg": [0.6440718173980713], "cos_innovation_negg": [0.6440718173980713], "cos_apical_negg": [0.6440718173980713], "cos_Ac_negg": [0.6440718173980713], "r_norm": [0.20286446809768677]}, {"step": 2950, "epoch": 7, "train_loss": 0.0025176741182804108, "cos_r_negg": [0.650090217590332], "cos_innovation_negg": [0.650090217590332], "cos_apical_negg": [0.650090217590332], "cos_Ac_negg": [0.650090217590332], "r_norm": [0.20614682137966156]}, {"step": 3000, "epoch": 7, "train_loss": 0.005629417486488819, "cos_r_negg": [0.6479119062423706], "cos_innovation_negg": [0.6479119062423706], "cos_apical_negg": [0.6479119062423706], "cos_Ac_negg": [0.6479119062423706], "r_norm": [0.2127581387758255], "ldr": {"dL_sdil": -0.0034825094044208527, "dL_bp": -0.0035923197865486145, "ratio": 0.9694319023214623}}, {"step": 3050, "epoch": 7, "train_loss": 0.005072558764368296, "cos_r_negg": [0.6483415365219116], "cos_innovation_negg": [0.6483415365219116], "cos_apical_negg": [0.6483415365219116], "cos_Ac_negg": [0.6483415365219116], "r_norm": [0.20213374495506287]}, {"step": 3100, "epoch": 7, "train_loss": 0.005258928053081036, "cos_r_negg": [0.6499826908111572], "cos_innovation_negg": [0.6499826908111572], "cos_apical_negg": [0.6499826908111572], "cos_Ac_negg": [0.6499826908111572], "r_norm": [0.18239551782608032]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9751, "test_loss": 0.08846557369232178}, {"step": 3150, "epoch": 8, "train_loss": 0.004414690192788839, "cos_r_negg": [0.6477656364440918], "cos_innovation_negg": [0.6477656364440918], "cos_apical_negg": [0.6477656364440918], "cos_Ac_negg": [0.6477656364440918], "r_norm": [0.20421180129051208]}, {"step": 3200, "epoch": 8, "train_loss": 0.007037022151052952, "cos_r_negg": [0.6489906907081604], "cos_innovation_negg": [0.6489906907081604], "cos_apical_negg": [0.6489906907081604], "cos_Ac_negg": [0.6489906907081604], "r_norm": [0.19996652007102966]}, {"step": 3250, "epoch": 8, "train_loss": 0.003335286397486925, "cos_r_negg": [0.6488131284713745], "cos_innovation_negg": [0.6488131284713745], "cos_apical_negg": [0.6488131284713745], "cos_Ac_negg": [0.6488131284713745], "r_norm": [0.1973746418952942], "ldr": {"dL_sdil": -0.0030824951827526093, "dL_bp": -0.003016870468854904, "ratio": 1.0217525792291022}}, {"step": 3300, "epoch": 8, "train_loss": 0.0036672756541520357, "cos_r_negg": [0.6504677534103394], "cos_innovation_negg": [0.6504677534103394], "cos_apical_negg": [0.6504677534103394], "cos_Ac_negg": [0.6504677534103394], "r_norm": [0.2063758224248886]}, {"step": 3350, "epoch": 8, "train_loss": 0.01492753904312849, "cos_r_negg": [0.6521838903427124], "cos_innovation_negg": [0.6521838903427124], "cos_apical_negg": [0.6521838903427124], "cos_Ac_negg": [0.6521838903427124], "r_norm": [0.18838709592819214]}, {"step": 3400, "epoch": 8, "train_loss": 0.004698086995631456, "cos_r_negg": [0.64848393201828], "cos_innovation_negg": [0.64848393201828], "cos_apical_negg": [0.64848393201828], "cos_Ac_negg": [0.64848393201828], "r_norm": [0.19895412027835846]}, {"step": 3450, "epoch": 8, "train_loss": 0.015807535499334335, "cos_r_negg": [0.652592658996582], "cos_innovation_negg": [0.652592658996582], "cos_apical_negg": [0.652592658996582], "cos_Ac_negg": [0.652592658996582], "r_norm": [0.20557649433612823]}, {"step": 3500, "epoch": 8, "train_loss": 0.009331470355391502, "cos_r_negg": [0.6547179222106934], "cos_innovation_negg": [0.6547179222106934], "cos_apical_negg": [0.6547179222106934], "cos_Ac_negg": [0.6547179222106934], "r_norm": [0.20355701446533203], "ldr": {"dL_sdil": -0.0034710727632045746, "dL_bp": -0.003576897084712982, "ratio": 0.9704144908276278}}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9747, "test_loss": 0.08839641704559326}, {"step": 3550, "epoch": 9, "train_loss": 0.0032112812623381615, "cos_r_negg": [0.6529762148857117], "cos_innovation_negg": [0.6529762148857117], "cos_apical_negg": [0.6529762148857117], "cos_Ac_negg": [0.6529762148857117], "r_norm": [0.20045408606529236]}, {"step": 3600, "epoch": 9, "train_loss": 0.0034488472156226635, "cos_r_negg": [0.6520247459411621], "cos_innovation_negg": [0.6520247459411621], "cos_apical_negg": [0.6520247459411621], "cos_Ac_negg": [0.6520247459411621], "r_norm": [0.19480332732200623]}, {"step": 3650, "epoch": 9, "train_loss": 0.006094988901168108, "cos_r_negg": [0.6558542251586914], "cos_innovation_negg": [0.6558542251586914], "cos_apical_negg": [0.6558542251586914], "cos_Ac_negg": [0.6558542251586914], "r_norm": [0.18589213490486145]}, {"step": 3700, "epoch": 9, "train_loss": 0.006265830248594284, "cos_r_negg": [0.6564699411392212], "cos_innovation_negg": [0.6564699411392212], "cos_apical_negg": [0.6564699411392212], "cos_Ac_negg": [0.6564699411392212], "r_norm": [0.18831568956375122]}, {"step": 3750, "epoch": 9, "train_loss": 0.005002729594707489, "cos_r_negg": [0.655595600605011], "cos_innovation_negg": [0.655595600605011], "cos_apical_negg": [0.655595600605011], "cos_Ac_negg": [0.655595600605011], "r_norm": [0.19380924105644226], "ldr": {"dL_sdil": -0.0031746700406074524, "dL_bp": -0.0032497569918632507, "ratio": 0.9768945950593225}}, {"step": 3800, "epoch": 9, "train_loss": 0.0020509900059551, "cos_r_negg": [0.6574457883834839], "cos_innovation_negg": [0.6574457883834839], "cos_apical_negg": [0.6574457883834839], "cos_Ac_negg": [0.6574457883834839], "r_norm": [0.20294073224067688]}, {"step": 3850, "epoch": 9, "train_loss": 0.003226958680897951, "cos_r_negg": [0.6611326932907104], "cos_innovation_negg": [0.6611326932907104], "cos_apical_negg": [0.6611326932907104], "cos_Ac_negg": [0.6611326932907104], "r_norm": [0.19813671708106995]}, {"step": 3900, "epoch": 9, "train_loss": 0.007771424017846584, "cos_r_negg": [0.6593209505081177], "cos_innovation_negg": [0.6593209505081177], "cos_apical_negg": [0.6593209505081177], "cos_Ac_negg": [0.6593209505081177], "r_norm": [0.19765135645866394]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9748, "test_loss": 0.08772453660964966}, {"step": 3950, "epoch": 10, "train_loss": 0.003116397187113762, "cos_r_negg": [0.6569539904594421], "cos_innovation_negg": [0.6569539904594421], "cos_apical_negg": [0.6569539904594421], "cos_Ac_negg": [0.6569539904594421], "r_norm": [0.19880643486976624]}, {"step": 4000, "epoch": 10, "train_loss": 0.0026373867876827717, "cos_r_negg": [0.6590050458908081], "cos_innovation_negg": [0.6590050458908081], "cos_apical_negg": [0.6590050458908081], "cos_Ac_negg": [0.6590050458908081], "r_norm": [0.19262820482254028], "ldr": {"dL_sdil": -0.0035692378878593445, "dL_bp": -0.0036004334688186646, "ratio": 0.9913356041072588}}, {"step": 4050, "epoch": 10, "train_loss": 0.0015484970062971115, "cos_r_negg": [0.6596846580505371], "cos_innovation_negg": [0.6596846580505371], "cos_apical_negg": [0.6596846580505371], "cos_Ac_negg": [0.6596846580505371], "r_norm": [0.1932370960712433]}, {"step": 4100, "epoch": 10, "train_loss": 0.0025846525095403194, "cos_r_negg": [0.659912645816803], "cos_innovation_negg": [0.659912645816803], "cos_apical_negg": [0.659912645816803], "cos_Ac_negg": [0.659912645816803], "r_norm": [0.1921229064464569]}, {"step": 4150, "epoch": 10, "train_loss": 0.0013767535565420985, "cos_r_negg": [0.659926176071167], "cos_innovation_negg": [0.659926176071167], "cos_apical_negg": [0.659926176071167], "cos_Ac_negg": [0.659926176071167], "r_norm": [0.18835411965847015]}, {"step": 4200, "epoch": 10, "train_loss": 0.005181065760552883, "cos_r_negg": [0.6616376638412476], "cos_innovation_negg": [0.6616376638412476], "cos_apical_negg": [0.6616376638412476], "cos_Ac_negg": [0.6616376638412476], "r_norm": [0.1874384880065918]}, {"step": 4250, "epoch": 10, "train_loss": 0.007394136395305395, "cos_r_negg": [0.6639987230300903], "cos_innovation_negg": [0.6639987230300903], "cos_apical_negg": [0.6639987230300903], "cos_Ac_negg": [0.6639987230300903], "r_norm": [0.1924438178539276], "ldr": {"dL_sdil": -0.0033907517790794373, "dL_bp": -0.0034468621015548706, "ratio": 0.9837213323822493}}, {"step": 4300, "epoch": 10, "train_loss": 0.0024852536153048277, "cos_r_negg": [0.6627053022384644], "cos_innovation_negg": [0.6627053022384644], "cos_apical_negg": [0.6627053022384644], "cos_Ac_negg": [0.6627053022384644], "r_norm": [0.19183456897735596]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9744, "test_loss": 0.089670791721344}, {"step": 4350, "epoch": 11, "train_loss": 0.004016684368252754, "cos_r_negg": [0.6633303165435791], "cos_innovation_negg": [0.6633303165435791], "cos_apical_negg": [0.6633303165435791], "cos_Ac_negg": [0.6633303165435791], "r_norm": [0.19329418241977692]}, {"step": 4400, "epoch": 11, "train_loss": 0.001495786476880312, "cos_r_negg": [0.6611989140510559], "cos_innovation_negg": [0.6611989140510559], "cos_apical_negg": [0.6611989140510559], "cos_Ac_negg": [0.6611989140510559], "r_norm": [0.19347508251667023]}, {"step": 4450, "epoch": 11, "train_loss": 0.0039836629293859005, "cos_r_negg": [0.6619336605072021], "cos_innovation_negg": [0.6619336605072021], "cos_apical_negg": [0.6619336605072021], "cos_Ac_negg": [0.6619336605072021], "r_norm": [0.19052177667617798]}, {"step": 4500, "epoch": 11, "train_loss": 0.0007587734726257622, "cos_r_negg": [0.6630107760429382], "cos_innovation_negg": [0.6630107760429382], "cos_apical_negg": [0.6630107760429382], "cos_Ac_negg": [0.6630107760429382], "r_norm": [0.19042092561721802], "ldr": {"dL_sdil": -0.0033354274928569794, "dL_bp": -0.0034304670989513397, "ratio": 0.9722954328512834}}, {"step": 4550, "epoch": 11, "train_loss": 0.005601126700639725, "cos_r_negg": [0.661663293838501], "cos_innovation_negg": [0.661663293838501], "cos_apical_negg": [0.661663293838501], "cos_Ac_negg": [0.661663293838501], "r_norm": [0.18935249745845795]}, {"step": 4600, "epoch": 11, "train_loss": 0.001419921638444066, "cos_r_negg": [0.6637364029884338], "cos_innovation_negg": [0.6637364029884338], "cos_apical_negg": [0.6637364029884338], "cos_Ac_negg": [0.6637364029884338], "r_norm": [0.1854950487613678]}, {"step": 4650, "epoch": 11, "train_loss": 0.00275788689032197, "cos_r_negg": [0.6648856401443481], "cos_innovation_negg": [0.6648856401443481], "cos_apical_negg": [0.6648856401443481], "cos_Ac_negg": [0.6648856401443481], "r_norm": [0.19034051895141602]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9752, "test_loss": 0.08995066747665405}, {"step": 4700, "epoch": 12, "train_loss": 0.00309376185759902, "cos_r_negg": [0.6643451452255249], "cos_innovation_negg": [0.6643451452255249], "cos_apical_negg": [0.6643451452255249], "cos_Ac_negg": [0.6643451452255249], "r_norm": [0.19332051277160645]}, {"step": 4750, "epoch": 12, "train_loss": 0.0022119139321148396, "cos_r_negg": [0.6635880470275879], "cos_innovation_negg": [0.6635880470275879], "cos_apical_negg": [0.6635880470275879], "cos_Ac_negg": [0.6635880470275879], "r_norm": [0.19112156331539154], "ldr": {"dL_sdil": -0.0035537518560886383, "dL_bp": -0.003675796091556549, "ratio": 0.9667978765883528}}, {"step": 4800, "epoch": 12, "train_loss": 0.0022852004040032625, "cos_r_negg": [0.6648796796798706], "cos_innovation_negg": [0.6648796796798706], "cos_apical_negg": [0.6648796796798706], "cos_Ac_negg": [0.6648796796798706], "r_norm": [0.18916934728622437]}, {"step": 4850, "epoch": 12, "train_loss": 0.001951591926626861, "cos_r_negg": [0.6651219725608826], "cos_innovation_negg": [0.6651219725608826], "cos_apical_negg": [0.6651219725608826], "cos_Ac_negg": [0.6651219725608826], "r_norm": [0.1873171329498291]}, {"step": 4900, "epoch": 12, "train_loss": 0.0027105561457574368, "cos_r_negg": [0.6645501852035522], "cos_innovation_negg": [0.6645501852035522], "cos_apical_negg": [0.6645501852035522], "cos_Ac_negg": [0.6645501852035522], "r_norm": [0.18848127126693726]}, {"step": 4950, "epoch": 12, "train_loss": 0.0011318711331114173, "cos_r_negg": [0.6663046479225159], "cos_innovation_negg": [0.6663046479225159], "cos_apical_negg": [0.6663046479225159], "cos_Ac_negg": [0.6663046479225159], "r_norm": [0.18730175495147705]}, {"step": 5000, "epoch": 12, "train_loss": 0.0021451793145388365, "cos_r_negg": [0.6661809682846069], "cos_innovation_negg": [0.6661809682846069], "cos_apical_negg": [0.6661809682846069], "cos_Ac_negg": [0.6661809682846069], "r_norm": [0.18514122068881989], "ldr": {"dL_sdil": -0.003307037055492401, "dL_bp": -0.0034041628241539, "ratio": 0.9714685302441021}}, {"step": 5050, "epoch": 12, "train_loss": 0.0016976024489849806, "cos_r_negg": [0.6677188873291016], "cos_innovation_negg": [0.6677188873291016], "cos_apical_negg": [0.6677188873291016], "cos_Ac_negg": [0.6677188873291016], "r_norm": [0.18436864018440247]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9746, "test_loss": 0.09175202016830444}, {"step": 5100, "epoch": 13, "train_loss": 0.0018152374541386962, "cos_r_negg": [0.666205883026123], "cos_innovation_negg": [0.666205883026123], "cos_apical_negg": [0.666205883026123], "cos_Ac_negg": [0.666205883026123], "r_norm": [0.19294962286949158]}, {"step": 5150, "epoch": 13, "train_loss": 0.003475346602499485, "cos_r_negg": [0.6664069890975952], "cos_innovation_negg": [0.6664069890975952], "cos_apical_negg": [0.6664069890975952], "cos_Ac_negg": [0.6664069890975952], "r_norm": [0.18639397621154785]}, {"step": 5200, "epoch": 13, "train_loss": 0.0018608508398756385, "cos_r_negg": [0.6653834581375122], "cos_innovation_negg": [0.6653834581375122], "cos_apical_negg": [0.6653834581375122], "cos_Ac_negg": [0.6653834581375122], "r_norm": [0.19113487005233765]}, {"step": 5250, "epoch": 13, "train_loss": 0.0012236915063112974, "cos_r_negg": [0.6658716201782227], "cos_innovation_negg": [0.6658716201782227], "cos_apical_negg": [0.6658716201782227], "cos_Ac_negg": [0.6658716201782227], "r_norm": [0.18999069929122925], "ldr": {"dL_sdil": -0.0035007745027542114, "dL_bp": -0.0036511868238449097, "ratio": 0.9588045399078469}}, {"step": 5300, "epoch": 13, "train_loss": 0.0024335780180990696, "cos_r_negg": [0.666246771812439], "cos_innovation_negg": [0.666246771812439], "cos_apical_negg": [0.666246771812439], "cos_Ac_negg": [0.666246771812439], "r_norm": [0.19098010659217834]}, {"step": 5350, "epoch": 13, "train_loss": 0.0031038715969771147, "cos_r_negg": [0.666203498840332], "cos_innovation_negg": [0.666203498840332], "cos_apical_negg": [0.666203498840332], "cos_Ac_negg": [0.666203498840332], "r_norm": [0.19009268283843994]}, {"step": 5400, "epoch": 13, "train_loss": 0.0033635690342634916, "cos_r_negg": [0.6686925888061523], "cos_innovation_negg": [0.6686925888061523], "cos_apical_negg": [0.6686925888061523], "cos_Ac_negg": [0.6686925888061523], "r_norm": [0.1874905824661255]}, {"step": 5450, "epoch": 13, "train_loss": 0.0012686345726251602, "cos_r_negg": [0.6673544645309448], "cos_innovation_negg": [0.6673544645309448], "cos_apical_negg": [0.6673544645309448], "cos_Ac_negg": [0.6673544645309448], "r_norm": [0.18319319188594818]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9751, "test_loss": 0.09248029098510742}, {"step": 5500, "epoch": 14, "train_loss": 0.0011547560570761561, "cos_r_negg": [0.6661609411239624], "cos_innovation_negg": [0.6661609411239624], "cos_apical_negg": [0.6661609411239624], "cos_Ac_negg": [0.6661609411239624], "r_norm": [0.19202403724193573], "ldr": {"dL_sdil": -0.003587815910577774, "dL_bp": -0.003754936158657074, "ratio": 0.9554931852319244}}, {"step": 5550, "epoch": 14, "train_loss": 0.0009745231363922358, "cos_r_negg": [0.6676994562149048], "cos_innovation_negg": [0.6676994562149048], "cos_apical_negg": [0.6676994562149048], "cos_Ac_negg": [0.6676994562149048], "r_norm": [0.18245328962802887]}, {"step": 5600, "epoch": 14, "train_loss": 0.001786916283890605, "cos_r_negg": [0.6678341031074524], "cos_innovation_negg": [0.6678341031074524], "cos_apical_negg": [0.6678341031074524], "cos_Ac_negg": [0.6678341031074524], "r_norm": [0.18859194219112396]}, {"step": 5650, "epoch": 14, "train_loss": 0.0012264908291399479, "cos_r_negg": [0.6694937944412231], "cos_innovation_negg": [0.6694937944412231], "cos_apical_negg": [0.6694937944412231], "cos_Ac_negg": [0.6694937944412231], "r_norm": [0.18558205664157867]}, {"step": 5700, "epoch": 14, "train_loss": 0.0003902111784555018, "cos_r_negg": [0.6681622862815857], "cos_innovation_negg": [0.6681622862815857], "cos_apical_negg": [0.6681622862815857], "cos_Ac_negg": [0.6681622862815857], "r_norm": [0.18749463558197021]}, {"step": 5750, "epoch": 14, "train_loss": 0.0021597177255898714, "cos_r_negg": [0.6685847043991089], "cos_innovation_negg": [0.6685847043991089], "cos_apical_negg": [0.6685847043991089], "cos_Ac_negg": [0.6685847043991089], "r_norm": [0.1878083497285843], "ldr": {"dL_sdil": -0.003606930375099182, "dL_bp": -0.0038375630974769592, "ratio": 0.9399012559482322}}, {"step": 5800, "epoch": 14, "train_loss": 0.0029637301340699196, "cos_r_negg": [0.6679900288581848], "cos_innovation_negg": [0.6679900288581848], "cos_apical_negg": [0.6679900288581848], "cos_Ac_negg": [0.6679900288581848], "r_norm": [0.19051578640937805]}, {"step": 5850, "epoch": 14, "train_loss": 0.0025078910402953625, "cos_r_negg": [0.6681166887283325], "cos_innovation_negg": [0.6681166887283325], "cos_apical_negg": [0.6681166887283325], "cos_Ac_negg": [0.6681166887283325], "r_norm": [0.1835900992155075]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9749, "test_loss": 0.09206979217529297}, {"step": 5900, "epoch": 15, "train_loss": 0.0016107386909425259, "cos_r_negg": [0.6695368885993958], "cos_innovation_negg": [0.6695368885993958], "cos_apical_negg": [0.6695368885993958], "cos_Ac_negg": [0.6695368885993958], "r_norm": [0.18587350845336914]}, {"step": 5950, "epoch": 15, "train_loss": 0.0026672733947634697, "cos_r_negg": [0.6707133054733276], "cos_innovation_negg": [0.6707133054733276], "cos_apical_negg": [0.6707133054733276], "cos_Ac_negg": [0.6707133054733276], "r_norm": [0.1816197633743286]}, {"step": 6000, "epoch": 15, "train_loss": 0.0021926462650299072, "cos_r_negg": [0.6706957221031189], "cos_innovation_negg": [0.6706957221031189], "cos_apical_negg": [0.6706957221031189], "cos_Ac_negg": [0.6706957221031189], "r_norm": [0.18417629599571228], "ldr": {"dL_sdil": -0.0036102905869483948, "dL_bp": -0.003763914108276367, "ratio": 0.9591851681763476}}, {"step": 6050, "epoch": 15, "train_loss": 0.004678631201386452, "cos_r_negg": [0.6698333024978638], "cos_innovation_negg": [0.6698333024978638], "cos_apical_negg": [0.6698333024978638], "cos_Ac_negg": [0.6698333024978638], "r_norm": [0.1855161190032959]}, {"step": 6100, "epoch": 15, "train_loss": 0.002193327760323882, "cos_r_negg": [0.6698936223983765], "cos_innovation_negg": [0.6698936223983765], "cos_apical_negg": [0.6698936223983765], "cos_Ac_negg": [0.6698936223983765], "r_norm": [0.19182786345481873]}, {"step": 6150, "epoch": 15, "train_loss": 0.001425995142199099, "cos_r_negg": [0.6699593663215637], "cos_innovation_negg": [0.6699593663215637], "cos_apical_negg": [0.6699593663215637], "cos_Ac_negg": [0.6699593663215637], "r_norm": [0.18158045411109924]}, {"step": 6200, "epoch": 15, "train_loss": 0.0014772354625165462, "cos_r_negg": [0.6706821918487549], "cos_innovation_negg": [0.6706821918487549], "cos_apical_negg": [0.6706821918487549], "cos_Ac_negg": [0.6706821918487549], "r_norm": [0.18641549348831177]}, {"step": 6250, "epoch": 15, "train_loss": 0.0012118282029405236, "cos_r_negg": [0.6697180271148682], "cos_innovation_negg": [0.6697180271148682], "cos_apical_negg": [0.6697180271148682], "cos_Ac_negg": [0.6697180271148682], "r_norm": [0.18767692148685455], "ldr": {"dL_sdil": -0.0035717710852622986, "dL_bp": -0.0037838146090507507, "ratio": 0.943960382392612}}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9755, "test_loss": 0.09298608655929566}, {"step": 6300, "epoch": 16, "train_loss": 0.002702809404581785, "cos_r_negg": [0.6704621315002441], "cos_innovation_negg": [0.6704621315002441], "cos_apical_negg": [0.6704621315002441], "cos_Ac_negg": [0.6704621315002441], "r_norm": [0.1884043663740158]}, {"step": 6350, "epoch": 16, "train_loss": 0.0013074952876195312, "cos_r_negg": [0.670046329498291], "cos_innovation_negg": [0.670046329498291], "cos_apical_negg": [0.670046329498291], "cos_Ac_negg": [0.670046329498291], "r_norm": [0.18494729697704315]}, {"step": 6400, "epoch": 16, "train_loss": 0.0017672681715339422, "cos_r_negg": [0.6700473427772522], "cos_innovation_negg": [0.6700473427772522], "cos_apical_negg": [0.6700473427772522], "cos_Ac_negg": [0.6700473427772522], "r_norm": [0.1828645020723343]}, {"step": 6450, "epoch": 16, "train_loss": 0.0008366332622244954, "cos_r_negg": [0.6706515550613403], "cos_innovation_negg": [0.6706515550613403], "cos_apical_negg": [0.6706515550613403], "cos_Ac_negg": [0.6706515550613403], "r_norm": [0.18646997213363647]}, {"step": 6500, "epoch": 16, "train_loss": 0.0010278522968292236, "cos_r_negg": [0.6704522371292114], "cos_innovation_negg": [0.6704522371292114], "cos_apical_negg": [0.6704522371292114], "cos_Ac_negg": [0.6704522371292114], "r_norm": [0.18312697112560272], "ldr": {"dL_sdil": -0.003514811396598816, "dL_bp": -0.0037639737129211426, "ratio": 0.9338033856434781}}, {"step": 6550, "epoch": 16, "train_loss": 0.0021616006270051003, "cos_r_negg": [0.6701947450637817], "cos_innovation_negg": [0.6701947450637817], "cos_apical_negg": [0.6701947450637817], "cos_Ac_negg": [0.6701947450637817], "r_norm": [0.18587425351142883]}, {"step": 6600, "epoch": 16, "train_loss": 0.001324089476838708, "cos_r_negg": [0.6723578572273254], "cos_innovation_negg": [0.6723578572273254], "cos_apical_negg": [0.6723578572273254], "cos_Ac_negg": [0.6723578572273254], "r_norm": [0.18028807640075684]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9749, "test_loss": 0.09304358043670655}, {"step": 6650, "epoch": 17, "train_loss": 0.0011091396445408463, "cos_r_negg": [0.6691625118255615], "cos_innovation_negg": [0.6691625118255615], "cos_apical_negg": [0.6691625118255615], "cos_Ac_negg": [0.6691625118255615], "r_norm": [0.18484261631965637]}, {"step": 6700, "epoch": 17, "train_loss": 0.0006124647334218025, "cos_r_negg": [0.6697288751602173], "cos_innovation_negg": [0.6697288751602173], "cos_apical_negg": [0.6697288751602173], "cos_Ac_negg": [0.6697288751602173], "r_norm": [0.18394812941551208]}, {"step": 6750, "epoch": 17, "train_loss": 0.002211962593719363, "cos_r_negg": [0.6703906655311584], "cos_innovation_negg": [0.6703906655311584], "cos_apical_negg": [0.6703906655311584], "cos_Ac_negg": [0.6703906655311584], "r_norm": [0.1827685385942459], "ldr": {"dL_sdil": -0.0036517083644866943, "dL_bp": -0.0038550272583961487, "ratio": 0.9472587662080388}}, {"step": 6800, "epoch": 17, "train_loss": 0.0018674805760383606, "cos_r_negg": [0.6700780391693115], "cos_innovation_negg": [0.6700780391693115], "cos_apical_negg": [0.6700780391693115], "cos_Ac_negg": [0.6700780391693115], "r_norm": [0.18294468522071838]}, {"step": 6850, "epoch": 17, "train_loss": 0.0015525654889643192, "cos_r_negg": [0.6708592772483826], "cos_innovation_negg": [0.6708592772483826], "cos_apical_negg": [0.6708592772483826], "cos_Ac_negg": [0.6708592772483826], "r_norm": [0.18247413635253906]}, {"step": 6900, "epoch": 17, "train_loss": 0.0020800656639039516, "cos_r_negg": [0.6710057854652405], "cos_innovation_negg": [0.6710057854652405], "cos_apical_negg": [0.6710057854652405], "cos_Ac_negg": [0.6710057854652405], "r_norm": [0.1851978600025177]}, {"step": 6950, "epoch": 17, "train_loss": 0.002418211195617914, "cos_r_negg": [0.6707087755203247], "cos_innovation_negg": [0.6707087755203247], "cos_apical_negg": [0.6707087755203247], "cos_Ac_negg": [0.6707087755203247], "r_norm": [0.18484926223754883]}, {"step": 7000, "epoch": 17, "train_loss": 0.0026086594443768263, "cos_r_negg": [0.6703905463218689], "cos_innovation_negg": [0.6703905463218689], "cos_apical_negg": [0.6703905463218689], "cos_Ac_negg": [0.6703905463218689], "r_norm": [0.18203988671302795], "ldr": {"dL_sdil": -0.003647617995738983, "dL_bp": -0.003876626491546631, "ratio": 0.9409258291179139}}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9745, "test_loss": 0.09415543632507324}, {"step": 7050, "epoch": 18, "train_loss": 0.0007436972227878869, "cos_r_negg": [0.6717532277107239], "cos_innovation_negg": [0.6717532277107239], "cos_apical_negg": [0.6717532277107239], "cos_Ac_negg": [0.6717532277107239], "r_norm": [0.18298107385635376]}, {"step": 7100, "epoch": 18, "train_loss": 0.0020655575208365917, "cos_r_negg": [0.6714507341384888], "cos_innovation_negg": [0.6714507341384888], "cos_apical_negg": [0.6714507341384888], "cos_Ac_negg": [0.6714507341384888], "r_norm": [0.18778809905052185]}, {"step": 7150, "epoch": 18, "train_loss": 0.0006751002511009574, "cos_r_negg": [0.6722816228866577], "cos_innovation_negg": [0.6722816228866577], "cos_apical_negg": [0.6722816228866577], "cos_Ac_negg": [0.6722816228866577], "r_norm": [0.18387791514396667]}, {"step": 7200, "epoch": 18, "train_loss": 0.001651648199185729, "cos_r_negg": [0.6725150942802429], "cos_innovation_negg": [0.6725150942802429], "cos_apical_negg": [0.6725150942802429], "cos_Ac_negg": [0.6725150942802429], "r_norm": [0.17902331054210663]}, {"step": 7250, "epoch": 18, "train_loss": 0.0016688019968569279, "cos_r_negg": [0.6722551584243774], "cos_innovation_negg": [0.6722551584243774], "cos_apical_negg": [0.6722551584243774], "cos_Ac_negg": [0.6722551584243774], "r_norm": [0.18548163771629333], "ldr": {"dL_sdil": -0.003820560872554779, "dL_bp": -0.004069305956363678, "ratio": 0.9388728479804018}}, {"step": 7300, "epoch": 18, "train_loss": 0.0005514713120646775, "cos_r_negg": [0.6708295941352844], "cos_innovation_negg": [0.6708295941352844], "cos_apical_negg": [0.6708295941352844], "cos_Ac_negg": [0.6708295941352844], "r_norm": [0.18334239721298218]}, {"step": 7350, "epoch": 18, "train_loss": 0.0020514456555247307, "cos_r_negg": [0.672799825668335], "cos_innovation_negg": [0.672799825668335], "cos_apical_negg": [0.672799825668335], "cos_Ac_negg": [0.672799825668335], "r_norm": [0.17923761904239655]}, {"step": 7400, "epoch": 18, "train_loss": 0.0007911993307061493, "cos_r_negg": [0.6732131838798523], "cos_innovation_negg": [0.6732131838798523], "cos_apical_negg": [0.6732131838798523], "cos_Ac_negg": [0.6732131838798523], "r_norm": [0.17971353232860565]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9752, "test_loss": 0.09512609367370606}, {"step": 7450, "epoch": 19, "train_loss": 0.0007249515620060265, "cos_r_negg": [0.6715526580810547], "cos_innovation_negg": [0.6715526580810547], "cos_apical_negg": [0.6715526580810547], "cos_Ac_negg": [0.6715526580810547], "r_norm": [0.18105623126029968]}, {"step": 7500, "epoch": 19, "train_loss": 0.0008969079935923219, "cos_r_negg": [0.6706702709197998], "cos_innovation_negg": [0.6706702709197998], "cos_apical_negg": [0.6706702709197998], "cos_Ac_negg": [0.6706702709197998], "r_norm": [0.18245379626750946], "ldr": {"dL_sdil": -0.0037984661757946014, "dL_bp": -0.004038237035274506, "ratio": 0.9406248673900329}}, {"step": 7550, "epoch": 19, "train_loss": 0.0006949532544240355, "cos_r_negg": [0.6719008684158325], "cos_innovation_negg": [0.6719008684158325], "cos_apical_negg": [0.6719008684158325], "cos_Ac_negg": [0.6719008684158325], "r_norm": [0.17860335111618042]}, {"step": 7600, "epoch": 19, "train_loss": 0.0006747113075107336, "cos_r_negg": [0.6714373826980591], "cos_innovation_negg": [0.6714373826980591], "cos_apical_negg": [0.6714373826980591], "cos_Ac_negg": [0.6714373826980591], "r_norm": [0.18105155229568481]}, {"step": 7650, "epoch": 19, "train_loss": 0.00206524645909667, "cos_r_negg": [0.6720503568649292], "cos_innovation_negg": [0.6720503568649292], "cos_apical_negg": [0.6720503568649292], "cos_Ac_negg": [0.6720503568649292], "r_norm": [0.18440188467502594]}, {"step": 7700, "epoch": 19, "train_loss": 0.0023550596088171005, "cos_r_negg": [0.6718701124191284], "cos_innovation_negg": [0.6718701124191284], "cos_apical_negg": [0.6718701124191284], "cos_Ac_negg": [0.6718701124191284], "r_norm": [0.18530887365341187]}, {"step": 7750, "epoch": 19, "train_loss": 0.0011377695482224226, "cos_r_negg": [0.6713353395462036], "cos_innovation_negg": [0.6713353395462036], "cos_apical_negg": [0.6713353395462036], "cos_Ac_negg": [0.6713353395462036], "r_norm": [0.1826344132423401], "ldr": {"dL_sdil": -0.0038183927536010742, "dL_bp": -0.004056796431541443, "ratio": 0.9412335122150106}}, {"step": 7800, "epoch": 19, "train_loss": 0.0013620276004076004, "cos_r_negg": [0.6723531484603882], "cos_innovation_negg": [0.6723531484603882], "cos_apical_negg": [0.6723531484603882], "cos_Ac_negg": [0.6723531484603882], "r_norm": [0.18325470387935638]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9746, "test_loss": 0.09601678943634033}, {"step": 7850, "epoch": 20, "train_loss": 0.0017682209145277739, "cos_r_negg": [0.6727221012115479], "cos_innovation_negg": [0.6727221012115479], "cos_apical_negg": [0.6727221012115479], "cos_Ac_negg": [0.6727221012115479], "r_norm": [0.1791473925113678]}, {"step": 7900, "epoch": 20, "train_loss": 0.0019275869708508253, "cos_r_negg": [0.6729167699813843], "cos_innovation_negg": [0.6729167699813843], "cos_apical_negg": [0.6729167699813843], "cos_Ac_negg": [0.6729167699813843], "r_norm": [0.18233643472194672]}, {"step": 7950, "epoch": 20, "train_loss": 0.0008337795734405518, "cos_r_negg": [0.6723600029945374], "cos_innovation_negg": [0.6723600029945374], "cos_apical_negg": [0.6723600029945374], "cos_Ac_negg": [0.6723600029945374], "r_norm": [0.1794581264257431]}, {"step": 8000, "epoch": 20, "train_loss": 0.0014088513562455773, "cos_r_negg": [0.672518253326416], "cos_innovation_negg": [0.672518253326416], "cos_apical_negg": [0.672518253326416], "cos_Ac_negg": [0.672518253326416], "r_norm": [0.18185700476169586], "ldr": {"dL_sdil": -0.0037495046854019165, "dL_bp": -0.004041686654090881, "ratio": 0.927707911647919}}, {"step": 8050, "epoch": 20, "train_loss": 0.0016475780867040157, "cos_r_negg": [0.6728813648223877], "cos_innovation_negg": [0.6728813648223877], "cos_apical_negg": [0.6728813648223877], "cos_Ac_negg": [0.6728813648223877], "r_norm": [0.18053437769412994]}, {"step": 8100, "epoch": 20, "train_loss": 0.0011628717184066772, "cos_r_negg": [0.6730326414108276], "cos_innovation_negg": [0.6730326414108276], "cos_apical_negg": [0.6730326414108276], "cos_Ac_negg": [0.6730326414108276], "r_norm": [0.18136854469776154]}, {"step": 8150, "epoch": 20, "train_loss": 0.0017150228377431631, "cos_r_negg": [0.6730549335479736], "cos_innovation_negg": [0.6730549335479736], "cos_apical_negg": [0.6730549335479736], "cos_Ac_negg": [0.6730549335479736], "r_norm": [0.1823369860649109]}, {"step": 8200, "epoch": 20, "train_loss": 0.0010967866983264685, "cos_r_negg": [0.6738962531089783], "cos_innovation_negg": [0.6738962531089783], "cos_apical_negg": [0.6738962531089783], "cos_Ac_negg": [0.6738962531089783], "r_norm": [0.18357551097869873]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9749, "test_loss": 0.09571176376342773}, {"step": 8250, "epoch": 21, "train_loss": 0.0013602799735963345, "cos_r_negg": [0.6728267073631287], "cos_innovation_negg": [0.6728267073631287], "cos_apical_negg": [0.6728267073631287], "cos_Ac_negg": [0.6728267073631287], "r_norm": [0.1830824911594391], "ldr": {"dL_sdil": -0.0038352757692337036, "dL_bp": -0.0040917545557022095, "ratio": 0.9373181399380174}}, {"step": 8300, "epoch": 21, "train_loss": 0.0010917035397142172, "cos_r_negg": [0.6743839979171753], "cos_innovation_negg": [0.6743839979171753], "cos_apical_negg": [0.6743839979171753], "cos_Ac_negg": [0.6743839979171753], "r_norm": [0.18177852034568787]}, {"step": 8350, "epoch": 21, "train_loss": 0.0011703858617693186, "cos_r_negg": [0.6740772724151611], "cos_innovation_negg": [0.6740772724151611], "cos_apical_negg": [0.6740772724151611], "cos_Ac_negg": [0.6740772724151611], "r_norm": [0.18348321318626404]}, {"step": 8400, "epoch": 21, "train_loss": 0.0007806543726474047, "cos_r_negg": [0.6741630434989929], "cos_innovation_negg": [0.6741630434989929], "cos_apical_negg": [0.6741630434989929], "cos_Ac_negg": [0.6741630434989929], "r_norm": [0.1823076456785202]}, {"step": 8450, "epoch": 21, "train_loss": 0.0011760510969907045, "cos_r_negg": [0.674295961856842], "cos_innovation_negg": [0.674295961856842], "cos_apical_negg": [0.674295961856842], "cos_Ac_negg": [0.674295961856842], "r_norm": [0.18005183339118958]}, {"step": 8500, "epoch": 21, "train_loss": 0.0017886791611090302, "cos_r_negg": [0.6753990054130554], "cos_innovation_negg": [0.6753990054130554], "cos_apical_negg": [0.6753990054130554], "cos_Ac_negg": [0.6753990054130554], "r_norm": [0.1799861192703247], "ldr": {"dL_sdil": -0.00385071337223053, "dL_bp": -0.00412391871213913, "ratio": 0.9337510365797476}}, {"step": 8550, "epoch": 21, "train_loss": 0.0012125539360567927, "cos_r_negg": [0.6749388575553894], "cos_innovation_negg": [0.6749388575553894], "cos_apical_negg": [0.6749388575553894], "cos_Ac_negg": [0.6749388575553894], "r_norm": [0.1814495325088501]}, {"step": 8600, "epoch": 21, "train_loss": 0.0026013345923274755, "cos_r_negg": [0.6753969788551331], "cos_innovation_negg": [0.6753969788551331], "cos_apical_negg": [0.6753969788551331], "cos_Ac_negg": [0.6753969788551331], "r_norm": [0.1788909137248993]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9749, "test_loss": 0.09679196949005127}, {"step": 8650, "epoch": 22, "train_loss": 0.0006287232972681522, "cos_r_negg": [0.6742681264877319], "cos_innovation_negg": [0.6742681264877319], "cos_apical_negg": [0.6742681264877319], "cos_Ac_negg": [0.6742681264877319], "r_norm": [0.18148142099380493]}, {"step": 8700, "epoch": 22, "train_loss": 0.0012091693934053183, "cos_r_negg": [0.6755259037017822], "cos_innovation_negg": [0.6755259037017822], "cos_apical_negg": [0.6755259037017822], "cos_Ac_negg": [0.6755259037017822], "r_norm": [0.17885278165340424]}, {"step": 8750, "epoch": 22, "train_loss": 0.00103406747803092, "cos_r_negg": [0.6754529476165771], "cos_innovation_negg": [0.6754529476165771], "cos_apical_negg": [0.6754529476165771], "cos_Ac_negg": [0.6754529476165771], "r_norm": [0.1801569163799286], "ldr": {"dL_sdil": -0.0038732588291168213, "dL_bp": -0.004177317023277283, "ratio": 0.927212085540514}}, {"step": 8800, "epoch": 22, "train_loss": 0.0010899676708504558, "cos_r_negg": [0.6746752262115479], "cos_innovation_negg": [0.6746752262115479], "cos_apical_negg": [0.6746752262115479], "cos_Ac_negg": [0.6746752262115479], "r_norm": [0.17732323706150055]}, {"step": 8850, "epoch": 22, "train_loss": 0.0009318299125880003, "cos_r_negg": [0.6751152276992798], "cos_innovation_negg": [0.6751152276992798], "cos_apical_negg": [0.6751152276992798], "cos_Ac_negg": [0.6751152276992798], "r_norm": [0.18184295296669006]}, {"step": 8900, "epoch": 22, "train_loss": 0.0017829141579568386, "cos_r_negg": [0.6758016347885132], "cos_innovation_negg": [0.6758016347885132], "cos_apical_negg": [0.6758016347885132], "cos_Ac_negg": [0.6758016347885132], "r_norm": [0.18206949532032013]}, {"step": 8950, "epoch": 22, "train_loss": 0.001048354897648096, "cos_r_negg": [0.6749475002288818], "cos_innovation_negg": [0.6749475002288818], "cos_apical_negg": [0.6749475002288818], "cos_Ac_negg": [0.6749475002288818], "r_norm": [0.1806318759918213]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9748, "test_loss": 0.09672517023086548}, {"step": 9000, "epoch": 23, "train_loss": 0.0011073736241087317, "cos_r_negg": [0.6760402917861938], "cos_innovation_negg": [0.6760402917861938], "cos_apical_negg": [0.6760402917861938], "cos_Ac_negg": [0.6760402917861938], "r_norm": [0.18101921677589417], "ldr": {"dL_sdil": -0.0038575157523155212, "dL_bp": -0.004135578870773315, "ratio": 0.9327631929781577}}, {"step": 9050, "epoch": 23, "train_loss": 0.001316361827775836, "cos_r_negg": [0.6759347915649414], "cos_innovation_negg": [0.6759347915649414], "cos_apical_negg": [0.6759347915649414], "cos_Ac_negg": [0.6759347915649414], "r_norm": [0.17937231063842773]}, {"step": 9100, "epoch": 23, "train_loss": 0.000977381831035018, "cos_r_negg": [0.6759316921234131], "cos_innovation_negg": [0.6759316921234131], "cos_apical_negg": [0.6759316921234131], "cos_Ac_negg": [0.6759316921234131], "r_norm": [0.17941239476203918]}, {"step": 9150, "epoch": 23, "train_loss": 0.001385595416650176, "cos_r_negg": [0.6769012212753296], "cos_innovation_negg": [0.6769012212753296], "cos_apical_negg": [0.6769012212753296], "cos_Ac_negg": [0.6769012212753296], "r_norm": [0.1796320378780365]}, {"step": 9200, "epoch": 23, "train_loss": 0.001875532791018486, "cos_r_negg": [0.6768275499343872], "cos_innovation_negg": [0.6768275499343872], "cos_apical_negg": [0.6768275499343872], "cos_Ac_negg": [0.6768275499343872], "r_norm": [0.1805676370859146]}, {"step": 9250, "epoch": 23, "train_loss": 0.0012778290547430515, "cos_r_negg": [0.6752597689628601], "cos_innovation_negg": [0.6752597689628601], "cos_apical_negg": [0.6752597689628601], "cos_Ac_negg": [0.6752597689628601], "r_norm": [0.17964893579483032], "ldr": {"dL_sdil": -0.00391194224357605, "dL_bp": -0.004218496382236481, "ratio": 0.9273309466493111}}, {"step": 9300, "epoch": 23, "train_loss": 0.0010427363449707627, "cos_r_negg": [0.6753631234169006], "cos_innovation_negg": [0.6753631234169006], "cos_apical_negg": [0.6753631234169006], "cos_Ac_negg": [0.6753631234169006], "r_norm": [0.18008559942245483]}, {"step": 9350, "epoch": 23, "train_loss": 0.0008182552992366254, "cos_r_negg": [0.6764692068099976], "cos_innovation_negg": [0.6764692068099976], "cos_apical_negg": [0.6764692068099976], "cos_Ac_negg": [0.6764692068099976], "r_norm": [0.17845658957958221]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9753, "test_loss": 0.09731565561294556}, {"step": 9400, "epoch": 24, "train_loss": 0.0009697069763205945, "cos_r_negg": [0.6759960055351257], "cos_innovation_negg": [0.6759960055351257], "cos_apical_negg": [0.6759960055351257], "cos_Ac_negg": [0.6759960055351257], "r_norm": [0.1799590289592743]}, {"step": 9450, "epoch": 24, "train_loss": 0.001001900527626276, "cos_r_negg": [0.6758776307106018], "cos_innovation_negg": [0.6758776307106018], "cos_apical_negg": [0.6758776307106018], "cos_Ac_negg": [0.6758776307106018], "r_norm": [0.17993758618831635]}, {"step": 9500, "epoch": 24, "train_loss": 0.0010743276216089725, "cos_r_negg": [0.6764757037162781], "cos_innovation_negg": [0.6764757037162781], "cos_apical_negg": [0.6764757037162781], "cos_Ac_negg": [0.6764757037162781], "r_norm": [0.17808982729911804], "ldr": {"dL_sdil": -0.003834240138530731, "dL_bp": -0.0041273534297943115, "ratio": 0.9289827497815742}}, {"step": 9550, "epoch": 24, "train_loss": 0.0011451351456344128, "cos_r_negg": [0.6773372292518616], "cos_innovation_negg": [0.6773372292518616], "cos_apical_negg": [0.6773372292518616], "cos_Ac_negg": [0.6773372292518616], "r_norm": [0.1806063950061798]}, {"step": 9600, "epoch": 24, "train_loss": 0.0008518341928720474, "cos_r_negg": [0.6763127446174622], "cos_innovation_negg": [0.6763127446174622], "cos_apical_negg": [0.6763127446174622], "cos_Ac_negg": [0.6763127446174622], "r_norm": [0.17769965529441833]}, {"step": 9650, "epoch": 24, "train_loss": 0.0016946353716775775, "cos_r_negg": [0.6763644218444824], "cos_innovation_negg": [0.6763644218444824], "cos_apical_negg": [0.6763644218444824], "cos_Ac_negg": [0.6763644218444824], "r_norm": [0.17693045735359192]}, {"step": 9700, "epoch": 24, "train_loss": 0.0006799446418881416, "cos_r_negg": [0.6760977506637573], "cos_innovation_negg": [0.6760977506637573], "cos_apical_negg": [0.6760977506637573], "cos_Ac_negg": [0.6760977506637573], "r_norm": [0.1795676350593567]}, {"step": 9750, "epoch": 24, "train_loss": 0.0019986475817859173, "cos_r_negg": [0.6756279468536377], "cos_innovation_negg": [0.6756279468536377], "cos_apical_negg": [0.6756279468536377], "cos_Ac_negg": [0.6756279468536377], "r_norm": [0.18210271000862122], "ldr": {"dL_sdil": -0.00402379035949707, "dL_bp": -0.004365883767604828, "ratio": 0.921643949697856}}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9749, "test_loss": 0.09747457876205444}], "final": {"test_acc": 0.9749, "test_loss": 0.09747457876205444, "wall_s": 15.14875602722168, "cos_r_negg": [0.6756224632263184], "cos_innovation_negg": [0.6756224632263184], "cos_apical_negg": [0.6756224632263184], "cos_Ac_negg": [0.6756224632263184]}} \ No newline at end of file diff --git a/results/ep_match_v1_mnist_bp_w256_d3_s0.json b/results/ep_match_v1_mnist_bp_w256_d3_s0.json new file mode 100644 index 0000000..8a174dc --- /dev/null +++ b/results/ep_match_v1_mnist_bp_w256_d3_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_match_v1_mnist_bp_w256_d3_s0", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3990073204040527}, {"step": 50, "epoch": 0, "train_loss": 0.4002436399459839}, {"step": 100, "epoch": 0, "train_loss": 0.17812389135360718}, {"step": 150, "epoch": 0, "train_loss": 0.1840030550956726}, {"step": 200, "epoch": 0, "train_loss": 0.20500800013542175}, {"step": 250, "epoch": 0, "train_loss": 0.2556897699832916}, {"step": 300, "epoch": 0, "train_loss": 0.12000337988138199}, {"step": 350, "epoch": 0, "train_loss": 0.2012084424495697}, {"epoch_end": 0, "step": 391, "test_acc": 0.9622, "test_loss": 0.12489149627685547}, {"step": 400, "epoch": 1, "train_loss": 0.09804942458868027}, {"step": 450, "epoch": 1, "train_loss": 0.10001759976148605}, {"step": 500, "epoch": 1, "train_loss": 0.09569170325994492}, {"step": 550, "epoch": 1, "train_loss": 0.06942141801118851}, {"step": 600, "epoch": 1, "train_loss": 0.0768204927444458}, {"step": 650, "epoch": 1, "train_loss": 0.061851054430007935}, {"step": 700, "epoch": 1, "train_loss": 0.07820960134267807}, {"step": 750, "epoch": 1, "train_loss": 0.1261262595653534}, {"epoch_end": 1, "step": 782, "test_acc": 0.9654, "test_loss": 0.11467089881896973}, {"step": 800, "epoch": 2, "train_loss": 0.05116385221481323}, {"step": 850, "epoch": 2, "train_loss": 0.046324390918016434}, {"step": 900, "epoch": 2, "train_loss": 0.05610174685716629}, {"step": 950, "epoch": 2, "train_loss": 0.06476321071386337}, {"step": 1000, "epoch": 2, "train_loss": 0.027614569291472435}, {"step": 1050, "epoch": 2, "train_loss": 0.030282452702522278}, {"step": 1100, "epoch": 2, "train_loss": 0.13309308886528015}, {"step": 1150, "epoch": 2, "train_loss": 0.05104924738407135}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9701, "test_loss": 0.09878145141601563}, {"step": 1200, "epoch": 3, "train_loss": 0.04369868338108063}, {"step": 1250, "epoch": 3, "train_loss": 0.059486910700798035}, {"step": 1300, "epoch": 3, "train_loss": 0.03649898245930672}, {"step": 1350, "epoch": 3, "train_loss": 0.022122176364064217}, {"step": 1400, "epoch": 3, "train_loss": 0.009704208001494408}, {"step": 1450, "epoch": 3, "train_loss": 0.00958540290594101}, {"step": 1500, "epoch": 3, "train_loss": 0.08372747153043747}, {"step": 1550, "epoch": 3, "train_loss": 0.02610473334789276}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9681, "test_loss": 0.1106717098236084}, {"step": 1600, "epoch": 4, "train_loss": 0.04812175780534744}, {"step": 1650, "epoch": 4, "train_loss": 0.09273818135261536}, {"step": 1700, "epoch": 4, "train_loss": 0.03223549574613571}, {"step": 1750, "epoch": 4, "train_loss": 0.0070333159528672695}, {"step": 1800, "epoch": 4, "train_loss": 0.020825661718845367}, {"step": 1850, "epoch": 4, "train_loss": 0.06324777752161026}, {"step": 1900, "epoch": 4, "train_loss": 0.04041361063718796}, {"step": 1950, "epoch": 4, "train_loss": 0.03099694289267063}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9781, "test_loss": 0.07957793445587158}, {"step": 2000, "epoch": 5, "train_loss": 0.06366223841905594}, {"step": 2050, "epoch": 5, "train_loss": 0.018138987943530083}, {"step": 2100, "epoch": 5, "train_loss": 0.009258276782929897}, {"step": 2150, "epoch": 5, "train_loss": 0.003421161789447069}, {"step": 2200, "epoch": 5, "train_loss": 0.012944572605192661}, {"step": 2250, "epoch": 5, "train_loss": 0.0035931868478655815}, {"step": 2300, "epoch": 5, "train_loss": 0.046689849346876144}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9781, "test_loss": 0.08209086294174195}, {"step": 2350, "epoch": 6, "train_loss": 0.005446103401482105}, {"step": 2400, "epoch": 6, "train_loss": 0.0018953274702653289}, {"step": 2450, "epoch": 6, "train_loss": 0.002217135624960065}, {"step": 2500, "epoch": 6, "train_loss": 0.008744921535253525}, {"step": 2550, "epoch": 6, "train_loss": 0.00392832700163126}, {"step": 2600, "epoch": 6, "train_loss": 0.0072005921974778175}, {"step": 2650, "epoch": 6, "train_loss": 0.005278072319924831}, {"step": 2700, "epoch": 6, "train_loss": 0.00730725284665823}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9771, "test_loss": 0.08509860572814941}, {"step": 2750, "epoch": 7, "train_loss": 0.000508412136696279}, {"step": 2800, "epoch": 7, "train_loss": 0.003539507044479251}, {"step": 2850, "epoch": 7, "train_loss": 0.0008909034077078104}, {"step": 2900, "epoch": 7, "train_loss": 0.0020411766599863768}, {"step": 2950, "epoch": 7, "train_loss": 0.0009922945173457265}, {"step": 3000, "epoch": 7, "train_loss": 0.0012641772627830505}, {"step": 3050, "epoch": 7, "train_loss": 0.0013533961027860641}, {"step": 3100, "epoch": 7, "train_loss": 0.005095204338431358}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9774, "test_loss": 0.09060556621551513}, {"step": 3150, "epoch": 8, "train_loss": 0.004277659580111504}, {"step": 3200, "epoch": 8, "train_loss": 0.003666764358058572}, {"step": 3250, "epoch": 8, "train_loss": 0.0010443925857543945}, {"step": 3300, "epoch": 8, "train_loss": 0.0019763587042689323}, {"step": 3350, "epoch": 8, "train_loss": 0.002213386818766594}, {"step": 3400, "epoch": 8, "train_loss": 0.0016309793572872877}, {"step": 3450, "epoch": 8, "train_loss": 0.0008183192112483084}, {"step": 3500, "epoch": 8, "train_loss": 0.0007308605127036572}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9804, "test_loss": 0.07836012649536132}, {"step": 3550, "epoch": 9, "train_loss": 0.0005177551647648215}, {"step": 3600, "epoch": 9, "train_loss": 0.003817811841145158}, {"step": 3650, "epoch": 9, "train_loss": 0.0009639505296945572}, {"step": 3700, "epoch": 9, "train_loss": 0.0006091279210522771}, {"step": 3750, "epoch": 9, "train_loss": 0.0005427014548331499}, {"step": 3800, "epoch": 9, "train_loss": 0.00037216267082840204}, {"step": 3850, "epoch": 9, "train_loss": 0.0008960545528680086}, {"step": 3900, "epoch": 9, "train_loss": 0.00043548201210796833}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9825, "test_loss": 0.07648620426654816}, {"step": 3950, "epoch": 10, "train_loss": 0.0004272467631381005}, {"step": 4000, "epoch": 10, "train_loss": 0.0004758799623232335}, {"step": 4050, "epoch": 10, "train_loss": 0.0004304641915950924}, {"step": 4100, "epoch": 10, "train_loss": 0.0002862158289644867}, {"step": 4150, "epoch": 10, "train_loss": 0.00018882655422203243}, {"step": 4200, "epoch": 10, "train_loss": 0.00027390988543629646}, {"step": 4250, "epoch": 10, "train_loss": 0.0004366169741842896}, {"step": 4300, "epoch": 10, "train_loss": 0.0006564374780282378}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9824, "test_loss": 0.07787980074882507}, {"step": 4350, "epoch": 11, "train_loss": 0.0002530182246118784}, {"step": 4400, "epoch": 11, "train_loss": 0.00027457470423541963}, {"step": 4450, "epoch": 11, "train_loss": 0.0002710337284952402}, {"step": 4500, "epoch": 11, "train_loss": 5.2421684813452885e-05}, {"step": 4550, "epoch": 11, "train_loss": 0.0001325177145190537}, {"step": 4600, "epoch": 11, "train_loss": 0.00011035226634703577}, {"step": 4650, "epoch": 11, "train_loss": 0.00012004546442767605}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9827, "test_loss": 0.07880608463287353}, {"step": 4700, "epoch": 12, "train_loss": 0.0002216714492533356}, {"step": 4750, "epoch": 12, "train_loss": 0.0003809183544944972}, {"step": 4800, "epoch": 12, "train_loss": 9.659333591116592e-05}, {"step": 4850, "epoch": 12, "train_loss": 0.0001721304579405114}, {"step": 4900, "epoch": 12, "train_loss": 0.0001387999946018681}, {"step": 4950, "epoch": 12, "train_loss": 0.00013610319001600146}, {"step": 5000, "epoch": 12, "train_loss": 0.00013637394295074046}, {"step": 5050, "epoch": 12, "train_loss": 0.00011306205851724371}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9826, "test_loss": 0.07953503336906433}, {"step": 5100, "epoch": 13, "train_loss": 5.017088551539928e-05}, {"step": 5150, "epoch": 13, "train_loss": 0.0001688714837655425}, {"step": 5200, "epoch": 13, "train_loss": 7.641459524165839e-05}, {"step": 5250, "epoch": 13, "train_loss": 9.812261851038784e-05}, {"step": 5300, "epoch": 13, "train_loss": 0.00020609537023119628}, {"step": 5350, "epoch": 13, "train_loss": 0.00027128448709845543}, {"step": 5400, "epoch": 13, "train_loss": 0.00022434399579651654}, {"step": 5450, "epoch": 13, "train_loss": 5.6483397202100605e-05}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9826, "test_loss": 0.07998231210708619}, {"step": 5500, "epoch": 14, "train_loss": 0.00015283537504728884}, {"step": 5550, "epoch": 14, "train_loss": 0.00017766986275091767}, {"step": 5600, "epoch": 14, "train_loss": 0.00010860081238206476}, {"step": 5650, "epoch": 14, "train_loss": 0.00010987577843479812}, {"step": 5700, "epoch": 14, "train_loss": 9.816438250709325e-05}, {"step": 5750, "epoch": 14, "train_loss": 0.00013830848911311477}, {"step": 5800, "epoch": 14, "train_loss": 0.0002125150931533426}, {"step": 5850, "epoch": 14, "train_loss": 0.00024908690829761326}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9825, "test_loss": 0.08059258830547333}, {"step": 5900, "epoch": 15, "train_loss": 0.00022079248446971178}, {"step": 5950, "epoch": 15, "train_loss": 0.00027873131330125034}, {"step": 6000, "epoch": 15, "train_loss": 0.0002764906093943864}, {"step": 6050, "epoch": 15, "train_loss": 0.0003720075765158981}, {"step": 6100, "epoch": 15, "train_loss": 0.00018380519759375602}, {"step": 6150, "epoch": 15, "train_loss": 0.0001699738495517522}, {"step": 6200, "epoch": 15, "train_loss": 0.00010869364632526413}, {"step": 6250, "epoch": 15, "train_loss": 5.590697765001096e-05}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9826, "test_loss": 0.08115480434894562}, {"step": 6300, "epoch": 16, "train_loss": 0.0004522484668996185}, {"step": 6350, "epoch": 16, "train_loss": 7.897536124801263e-05}, {"step": 6400, "epoch": 16, "train_loss": 0.00023459969088435173}, {"step": 6450, "epoch": 16, "train_loss": 8.418002107646316e-05}, {"step": 6500, "epoch": 16, "train_loss": 0.00011316806194372475}, {"step": 6550, "epoch": 16, "train_loss": 0.00017024522821884602}, {"step": 6600, "epoch": 16, "train_loss": 0.00011023900879081339}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9827, "test_loss": 0.08175262882709503}, {"step": 6650, "epoch": 17, "train_loss": 9.349813626613468e-05}, {"step": 6700, "epoch": 17, "train_loss": 7.408093370031565e-05}, {"step": 6750, "epoch": 17, "train_loss": 0.00020169615163467824}, {"step": 6800, "epoch": 17, "train_loss": 0.00016381533350795507}, {"step": 6850, "epoch": 17, "train_loss": 0.00010082448716275394}, {"step": 6900, "epoch": 17, "train_loss": 0.00021932146046310663}, {"step": 6950, "epoch": 17, "train_loss": 0.00022178447397891432}, {"step": 7000, "epoch": 17, "train_loss": 0.0002014718484133482}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9827, "test_loss": 0.08225314111709595}, {"step": 7050, "epoch": 18, "train_loss": 4.688418994192034e-05}, {"step": 7100, "epoch": 18, "train_loss": 0.00016482052160426974}, {"step": 7150, "epoch": 18, "train_loss": 0.00012172612332506105}, {"step": 7200, "epoch": 18, "train_loss": 0.00010427110828459263}, {"step": 7250, "epoch": 18, "train_loss": 0.00014771003043279052}, {"step": 7300, "epoch": 18, "train_loss": 8.625854388810694e-05}, {"step": 7350, "epoch": 18, "train_loss": 0.000193893036339432}, {"step": 7400, "epoch": 18, "train_loss": 7.09662344888784e-05}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9825, "test_loss": 0.08268591835498809}, {"step": 7450, "epoch": 19, "train_loss": 4.148980588070117e-05}, {"step": 7500, "epoch": 19, "train_loss": 0.00012566459190566093}, {"step": 7550, "epoch": 19, "train_loss": 4.990649540559389e-05}, {"step": 7600, "epoch": 19, "train_loss": 6.37869889033027e-05}, {"step": 7650, "epoch": 19, "train_loss": 0.00016114336904138327}, {"step": 7700, "epoch": 19, "train_loss": 0.0001768550428096205}, {"step": 7750, "epoch": 19, "train_loss": 0.0001308603532379493}, {"step": 7800, "epoch": 19, "train_loss": 0.00015564657223876566}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9824, "test_loss": 0.08314236409664154}, {"step": 7850, "epoch": 20, "train_loss": 0.00014971388736739755}, {"step": 7900, "epoch": 20, "train_loss": 0.00012795971997547895}, {"step": 7950, "epoch": 20, "train_loss": 7.135346822906286e-05}, {"step": 8000, "epoch": 20, "train_loss": 9.664152457844466e-05}, {"step": 8050, "epoch": 20, "train_loss": 0.00016255094669759274}, {"step": 8100, "epoch": 20, "train_loss": 9.142665658146143e-05}, {"step": 8150, "epoch": 20, "train_loss": 0.00011104050645371899}, {"step": 8200, "epoch": 20, "train_loss": 4.937179255648516e-05}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9826, "test_loss": 0.08326638298034668}, {"step": 8250, "epoch": 21, "train_loss": 0.00013000820763409138}, {"step": 8300, "epoch": 21, "train_loss": 0.00015429854101967067}, {"step": 8350, "epoch": 21, "train_loss": 0.00010806126374518499}, {"step": 8400, "epoch": 21, "train_loss": 5.06856304127723e-05}, {"step": 8450, "epoch": 21, "train_loss": 0.00024348469742108136}, {"step": 8500, "epoch": 21, "train_loss": 0.00013563015090767294}, {"step": 8550, "epoch": 21, "train_loss": 9.940647578332573e-05}, {"step": 8600, "epoch": 21, "train_loss": 0.0003000810684170574}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9822, "test_loss": 0.08370797386169433}, {"step": 8650, "epoch": 22, "train_loss": 5.3008156100986525e-05}, {"step": 8700, "epoch": 22, "train_loss": 7.81571798142977e-05}, {"step": 8750, "epoch": 22, "train_loss": 0.00010478337935637683}, {"step": 8800, "epoch": 22, "train_loss": 0.00020175515965092927}, {"step": 8850, "epoch": 22, "train_loss": 9.409543417859823e-05}, {"step": 8900, "epoch": 22, "train_loss": 0.000126328639453277}, {"step": 8950, "epoch": 22, "train_loss": 6.328766176011413e-05}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9826, "test_loss": 0.08409398782253265}, {"step": 9000, "epoch": 23, "train_loss": 6.655445758951828e-05}, {"step": 9050, "epoch": 23, "train_loss": 0.0001170139730675146}, {"step": 9100, "epoch": 23, "train_loss": 6.771195330657065e-05}, {"step": 9150, "epoch": 23, "train_loss": 9.408954792888835e-05}, {"step": 9200, "epoch": 23, "train_loss": 0.00010017631575465202}, {"step": 9250, "epoch": 23, "train_loss": 9.92634377325885e-05}, {"step": 9300, "epoch": 23, "train_loss": 7.477659528376535e-05}, {"step": 9350, "epoch": 23, "train_loss": 8.852800965541974e-05}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9824, "test_loss": 0.08449323916435242}, {"step": 9400, "epoch": 24, "train_loss": 0.00010905843373620883}, {"step": 9450, "epoch": 24, "train_loss": 0.00010415162250865251}, {"step": 9500, "epoch": 24, "train_loss": 0.0001084874093066901}, {"step": 9550, "epoch": 24, "train_loss": 9.476537525188178e-05}, {"step": 9600, "epoch": 24, "train_loss": 7.222766726044938e-05}, {"step": 9650, "epoch": 24, "train_loss": 0.00016842229524627328}, {"step": 9700, "epoch": 24, "train_loss": 5.5362303100991994e-05}, {"step": 9750, "epoch": 24, "train_loss": 0.00014930227189324796}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9824, "test_loss": 0.08480259132385254}], "final": {"test_acc": 0.9824, "test_loss": 0.08480259132385254, "wall_s": 22.140812158584595}} \ No newline at end of file diff --git a/results/ep_match_v1_mnist_bp_w256_d3_s1.json b/results/ep_match_v1_mnist_bp_w256_d3_s1.json new file mode 100644 index 0000000..df6270b --- /dev/null +++ b/results/ep_match_v1_mnist_bp_w256_d3_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_match_v1_mnist_bp_w256_d3_s1", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3325631618499756}, {"step": 50, "epoch": 0, "train_loss": 0.4062894880771637}, {"step": 100, "epoch": 0, "train_loss": 0.16361844539642334}, {"step": 150, "epoch": 0, "train_loss": 0.19673334062099457}, {"step": 200, "epoch": 0, "train_loss": 0.14791013300418854}, {"step": 250, "epoch": 0, "train_loss": 0.2407197803258896}, {"step": 300, "epoch": 0, "train_loss": 0.12002608925104141}, {"step": 350, "epoch": 0, "train_loss": 0.20369650423526764}, {"epoch_end": 0, "step": 391, "test_acc": 0.9617, "test_loss": 0.12612874488830567}, {"step": 400, "epoch": 1, "train_loss": 0.0750856027007103}, {"step": 450, "epoch": 1, "train_loss": 0.12193188071250916}, {"step": 500, "epoch": 1, "train_loss": 0.15592803061008453}, {"step": 550, "epoch": 1, "train_loss": 0.051551058888435364}, {"step": 600, "epoch": 1, "train_loss": 0.03788438066840172}, {"step": 650, "epoch": 1, "train_loss": 0.05099586397409439}, {"step": 700, "epoch": 1, "train_loss": 0.08039838075637817}, {"step": 750, "epoch": 1, "train_loss": 0.11641225963830948}, {"epoch_end": 1, "step": 782, "test_acc": 0.9544, "test_loss": 0.14191328277587892}, {"step": 800, "epoch": 2, "train_loss": 0.026418741792440414}, {"step": 850, "epoch": 2, "train_loss": 0.03117004781961441}, {"step": 900, "epoch": 2, "train_loss": 0.03376998007297516}, {"step": 950, "epoch": 2, "train_loss": 0.09083827584981918}, {"step": 1000, "epoch": 2, "train_loss": 0.08783892542123795}, {"step": 1050, "epoch": 2, "train_loss": 0.05772702023386955}, {"step": 1100, "epoch": 2, "train_loss": 0.10183035582304001}, {"step": 1150, "epoch": 2, "train_loss": 0.09193816781044006}, {"epoch_end": 2, "step": 1173, "test_acc": 0.971, "test_loss": 0.09404349174499511}, {"step": 1200, "epoch": 3, "train_loss": 0.04399043694138527}, {"step": 1250, "epoch": 3, "train_loss": 0.07507435977458954}, {"step": 1300, "epoch": 3, "train_loss": 0.01302655041217804}, {"step": 1350, "epoch": 3, "train_loss": 0.010821445845067501}, {"step": 1400, "epoch": 3, "train_loss": 0.035825345665216446}, {"step": 1450, "epoch": 3, "train_loss": 0.0366162545979023}, {"step": 1500, "epoch": 3, "train_loss": 0.08494264632463455}, {"step": 1550, "epoch": 3, "train_loss": 0.04154994711279869}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9699, "test_loss": 0.10227554550170899}, {"step": 1600, "epoch": 4, "train_loss": 0.014733505435287952}, {"step": 1650, "epoch": 4, "train_loss": 0.014943699352443218}, {"step": 1700, "epoch": 4, "train_loss": 0.011734668165445328}, {"step": 1750, "epoch": 4, "train_loss": 0.02340112254023552}, {"step": 1800, "epoch": 4, "train_loss": 0.00978026818484068}, {"step": 1850, "epoch": 4, "train_loss": 0.048281311988830566}, {"step": 1900, "epoch": 4, "train_loss": 0.009658155031502247}, {"step": 1950, "epoch": 4, "train_loss": 0.023045603185892105}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9769, "test_loss": 0.08383322887420654}, {"step": 2000, "epoch": 5, "train_loss": 0.03411373123526573}, {"step": 2050, "epoch": 5, "train_loss": 0.006264558527618647}, {"step": 2100, "epoch": 5, "train_loss": 0.012450381182134151}, {"step": 2150, "epoch": 5, "train_loss": 0.001777266850695014}, {"step": 2200, "epoch": 5, "train_loss": 0.007925473153591156}, {"step": 2250, "epoch": 5, "train_loss": 0.011860653758049011}, {"step": 2300, "epoch": 5, "train_loss": 0.0015712307067587972}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9791, "test_loss": 0.08084622478485107}, {"step": 2350, "epoch": 6, "train_loss": 0.004022595938295126}, {"step": 2400, "epoch": 6, "train_loss": 0.004246448166668415}, {"step": 2450, "epoch": 6, "train_loss": 0.007845284417271614}, {"step": 2500, "epoch": 6, "train_loss": 0.011849072761833668}, {"step": 2550, "epoch": 6, "train_loss": 0.0042687589302659035}, {"step": 2600, "epoch": 6, "train_loss": 0.0073865242302417755}, {"step": 2650, "epoch": 6, "train_loss": 0.0010338141582906246}, {"step": 2700, "epoch": 6, "train_loss": 0.0016179183730855584}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9771, "test_loss": 0.0932099946975708}, {"step": 2750, "epoch": 7, "train_loss": 0.010639828629791737}, {"step": 2800, "epoch": 7, "train_loss": 0.0032081424724310637}, {"step": 2850, "epoch": 7, "train_loss": 0.0012293326435610652}, {"step": 2900, "epoch": 7, "train_loss": 0.0015998403541743755}, {"step": 2950, "epoch": 7, "train_loss": 0.0003643273375928402}, {"step": 3000, "epoch": 7, "train_loss": 0.003539435099810362}, {"step": 3050, "epoch": 7, "train_loss": 0.005131750367581844}, {"step": 3100, "epoch": 7, "train_loss": 0.0011634033871814609}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9796, "test_loss": 0.08289039449691772}, {"step": 3150, "epoch": 8, "train_loss": 0.0099300816655159}, {"step": 3200, "epoch": 8, "train_loss": 0.004533491097390652}, {"step": 3250, "epoch": 8, "train_loss": 0.0009814684744924307}, {"step": 3300, "epoch": 8, "train_loss": 0.0008952064090408385}, {"step": 3350, "epoch": 8, "train_loss": 0.003215155564248562}, {"step": 3400, "epoch": 8, "train_loss": 0.0007949260179884732}, {"step": 3450, "epoch": 8, "train_loss": 0.0011700348695740104}, {"step": 3500, "epoch": 8, "train_loss": 0.0005982125294394791}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9824, "test_loss": 0.07400852632522582}, {"step": 3550, "epoch": 9, "train_loss": 0.0007954317843541503}, {"step": 3600, "epoch": 9, "train_loss": 0.000326495966874063}, {"step": 3650, "epoch": 9, "train_loss": 0.00042459016549400985}, {"step": 3700, "epoch": 9, "train_loss": 0.0003430017677601427}, {"step": 3750, "epoch": 9, "train_loss": 0.0004629908944480121}, {"step": 3800, "epoch": 9, "train_loss": 0.00038241350557655096}, {"step": 3850, "epoch": 9, "train_loss": 0.0004816525324713439}, {"step": 3900, "epoch": 9, "train_loss": 0.0006018519634380937}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9832, "test_loss": 0.07233229961395264}, {"step": 3950, "epoch": 10, "train_loss": 0.00027026355382986367}, {"step": 4000, "epoch": 10, "train_loss": 0.0003822089347522706}, {"step": 4050, "epoch": 10, "train_loss": 0.0002578026906121522}, {"step": 4100, "epoch": 10, "train_loss": 0.000534889695700258}, {"step": 4150, "epoch": 10, "train_loss": 0.00016956849140115082}, {"step": 4200, "epoch": 10, "train_loss": 0.0002398368378635496}, {"step": 4250, "epoch": 10, "train_loss": 0.0006342356209643185}, {"step": 4300, "epoch": 10, "train_loss": 0.0002546923642512411}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9836, "test_loss": 0.07202276310920715}, {"step": 4350, "epoch": 11, "train_loss": 0.00019828497897833586}, {"step": 4400, "epoch": 11, "train_loss": 0.0001799171295715496}, {"step": 4450, "epoch": 11, "train_loss": 0.0003385751915629953}, {"step": 4500, "epoch": 11, "train_loss": 8.028472075238824e-05}, {"step": 4550, "epoch": 11, "train_loss": 0.00044262828305363655}, {"step": 4600, "epoch": 11, "train_loss": 3.453751196502708e-05}, {"step": 4650, "epoch": 11, "train_loss": 0.00023965895525179803}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9836, "test_loss": 0.07243822321891785}, {"step": 4700, "epoch": 12, "train_loss": 0.0002773299056570977}, {"step": 4750, "epoch": 12, "train_loss": 0.00029453125898726285}, {"step": 4800, "epoch": 12, "train_loss": 0.00011935343354707584}, {"step": 4850, "epoch": 12, "train_loss": 0.0002621613966766745}, {"step": 4900, "epoch": 12, "train_loss": 7.082032243488356e-05}, {"step": 4950, "epoch": 12, "train_loss": 0.00023662181047257036}, {"step": 5000, "epoch": 12, "train_loss": 0.0001789689704310149}, {"step": 5050, "epoch": 12, "train_loss": 0.0002128922933479771}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9835, "test_loss": 0.07291243257522582}, {"step": 5100, "epoch": 13, "train_loss": 0.00021373145864345133}, {"step": 5150, "epoch": 13, "train_loss": 0.00021735948394052684}, {"step": 5200, "epoch": 13, "train_loss": 0.0001514983450761065}, {"step": 5250, "epoch": 13, "train_loss": 0.0001029448612825945}, {"step": 5300, "epoch": 13, "train_loss": 0.0002306865353602916}, {"step": 5350, "epoch": 13, "train_loss": 0.00020495204080361873}, {"step": 5400, "epoch": 13, "train_loss": 0.00021840236149728298}, {"step": 5450, "epoch": 13, "train_loss": 0.0002710320695769042}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9834, "test_loss": 0.07372501544952392}, {"step": 5500, "epoch": 14, "train_loss": 0.00010520913929212838}, {"step": 5550, "epoch": 14, "train_loss": 0.00011219426960451528}, {"step": 5600, "epoch": 14, "train_loss": 0.0001261958241229877}, {"step": 5650, "epoch": 14, "train_loss": 0.00012813140347134322}, {"step": 5700, "epoch": 14, "train_loss": 6.407888577086851e-05}, {"step": 5750, "epoch": 14, "train_loss": 0.00028403749456629157}, {"step": 5800, "epoch": 14, "train_loss": 0.0002136249386239797}, {"step": 5850, "epoch": 14, "train_loss": 0.00020806463726330549}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9836, "test_loss": 0.07381890320777892}, {"step": 5900, "epoch": 15, "train_loss": 0.00020334633882157505}, {"step": 5950, "epoch": 15, "train_loss": 0.00022471844567917287}, {"step": 6000, "epoch": 15, "train_loss": 0.0003669346042443067}, {"step": 6050, "epoch": 15, "train_loss": 0.00036742366501130164}, {"step": 6100, "epoch": 15, "train_loss": 0.00021419496624730527}, {"step": 6150, "epoch": 15, "train_loss": 0.00014094001380726695}, {"step": 6200, "epoch": 15, "train_loss": 0.00010928299161605537}, {"step": 6250, "epoch": 15, "train_loss": 0.00010344214388169348}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9835, "test_loss": 0.07423270087242126}, {"step": 6300, "epoch": 16, "train_loss": 0.00015978222654666752}, {"step": 6350, "epoch": 16, "train_loss": 0.0001320969604421407}, {"step": 6400, "epoch": 16, "train_loss": 0.0001574342604726553}, {"step": 6450, "epoch": 16, "train_loss": 9.357613453175873e-05}, {"step": 6500, "epoch": 16, "train_loss": 0.00018050971266347915}, {"step": 6550, "epoch": 16, "train_loss": 0.00020913399930577725}, {"step": 6600, "epoch": 16, "train_loss": 0.00011081439879490063}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9837, "test_loss": 0.07457496104240417}, {"step": 6650, "epoch": 17, "train_loss": 0.00010578301589703187}, {"step": 6700, "epoch": 17, "train_loss": 7.781591557431966e-05}, {"step": 6750, "epoch": 17, "train_loss": 0.0002153556706616655}, {"step": 6800, "epoch": 17, "train_loss": 0.00011402632662793621}, {"step": 6850, "epoch": 17, "train_loss": 0.00013057375326752663}, {"step": 6900, "epoch": 17, "train_loss": 0.0001555470225866884}, {"step": 6950, "epoch": 17, "train_loss": 0.00017181759176310152}, {"step": 7000, "epoch": 17, "train_loss": 0.00012253172462806106}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9837, "test_loss": 0.07508152170181275}, {"step": 7050, "epoch": 18, "train_loss": 8.097302634268999e-05}, {"step": 7100, "epoch": 18, "train_loss": 0.00022668969177175313}, {"step": 7150, "epoch": 18, "train_loss": 5.574307942879386e-05}, {"step": 7200, "epoch": 18, "train_loss": 0.0001350522506982088}, {"step": 7250, "epoch": 18, "train_loss": 0.00014387955889105797}, {"step": 7300, "epoch": 18, "train_loss": 0.0001235245435964316}, {"step": 7350, "epoch": 18, "train_loss": 0.00020811636932194233}, {"step": 7400, "epoch": 18, "train_loss": 9.147117816610262e-05}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9836, "test_loss": 0.07537238264083862}, {"step": 7450, "epoch": 19, "train_loss": 5.098049223306589e-05}, {"step": 7500, "epoch": 19, "train_loss": 9.744838462211192e-05}, {"step": 7550, "epoch": 19, "train_loss": 8.816841727821156e-05}, {"step": 7600, "epoch": 19, "train_loss": 5.165467882761732e-05}, {"step": 7650, "epoch": 19, "train_loss": 0.00015488021017517895}, {"step": 7700, "epoch": 19, "train_loss": 0.00017741270130500197}, {"step": 7750, "epoch": 19, "train_loss": 0.00019727423205040395}, {"step": 7800, "epoch": 19, "train_loss": 0.00018003073637373745}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9836, "test_loss": 0.07582477393150329}, {"step": 7850, "epoch": 20, "train_loss": 0.00019321487343404442}, {"step": 7900, "epoch": 20, "train_loss": 0.00018533786351326853}, {"step": 7950, "epoch": 20, "train_loss": 7.065190584398806e-05}, {"step": 8000, "epoch": 20, "train_loss": 0.00010229471081402153}, {"step": 8050, "epoch": 20, "train_loss": 0.00011892181646544486}, {"step": 8100, "epoch": 20, "train_loss": 9.957268775906414e-05}, {"step": 8150, "epoch": 20, "train_loss": 9.94550937321037e-05}, {"step": 8200, "epoch": 20, "train_loss": 6.204706005519256e-05}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9836, "test_loss": 0.07608332772254944}, {"step": 8250, "epoch": 21, "train_loss": 0.00014569082122761756}, {"step": 8300, "epoch": 21, "train_loss": 0.00019479138427414}, {"step": 8350, "epoch": 21, "train_loss": 8.35837927297689e-05}, {"step": 8400, "epoch": 21, "train_loss": 7.380277384072542e-05}, {"step": 8450, "epoch": 21, "train_loss": 0.00014428199210669845}, {"step": 8500, "epoch": 21, "train_loss": 0.00019005377544090152}, {"step": 8550, "epoch": 21, "train_loss": 0.00011364032252458856}, {"step": 8600, "epoch": 21, "train_loss": 0.0002783231029752642}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9834, "test_loss": 0.07639921193122864}, {"step": 8650, "epoch": 22, "train_loss": 6.6893502662424e-05}, {"step": 8700, "epoch": 22, "train_loss": 9.31581889744848e-05}, {"step": 8750, "epoch": 22, "train_loss": 0.00011822923261206597}, {"step": 8800, "epoch": 22, "train_loss": 0.00016054930165410042}, {"step": 8850, "epoch": 22, "train_loss": 0.00013832935655955225}, {"step": 8900, "epoch": 22, "train_loss": 0.00016712347860448062}, {"step": 8950, "epoch": 22, "train_loss": 6.333515921141952e-05}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9835, "test_loss": 0.07659311084747314}, {"step": 9000, "epoch": 23, "train_loss": 7.244045991683379e-05}, {"step": 9050, "epoch": 23, "train_loss": 8.404106483794749e-05}, {"step": 9100, "epoch": 23, "train_loss": 8.997942495625466e-05}, {"step": 9150, "epoch": 23, "train_loss": 0.00010577197099337354}, {"step": 9200, "epoch": 23, "train_loss": 0.00018706006812863052}, {"step": 9250, "epoch": 23, "train_loss": 8.979786070995033e-05}, {"step": 9300, "epoch": 23, "train_loss": 0.00010404802742414176}, {"step": 9350, "epoch": 23, "train_loss": 8.213470573537052e-05}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9836, "test_loss": 0.07696412167549134}, {"step": 9400, "epoch": 24, "train_loss": 7.154409831855446e-05}, {"step": 9450, "epoch": 24, "train_loss": 5.5991356930462644e-05}, {"step": 9500, "epoch": 24, "train_loss": 9.039654833031818e-05}, {"step": 9550, "epoch": 24, "train_loss": 7.584375271108001e-05}, {"step": 9600, "epoch": 24, "train_loss": 0.00013271384523250163}, {"step": 9650, "epoch": 24, "train_loss": 0.00018520215235184878}, {"step": 9700, "epoch": 24, "train_loss": 6.0158719861647114e-05}, {"step": 9750, "epoch": 24, "train_loss": 0.0001315076369792223}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9835, "test_loss": 0.07714285397529602}], "final": {"test_acc": 0.9835, "test_loss": 0.07714285397529602, "wall_s": 22.587456941604614}} \ No newline at end of file diff --git a/results/ep_match_v1_mnist_dfa_w256_d3_s0.json b/results/ep_match_v1_mnist_dfa_w256_d3_s0.json new file mode 100644 index 0000000..35ac4b7 --- /dev/null +++ b/results/ep_match_v1_mnist_dfa_w256_d3_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_match_v1_mnist_dfa_w256_d3_s0", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3990068435668945, "cos_r_negg": [-0.019359100610017776, -0.02121054008603096, 0.03364589810371399], "cos_innovation_negg": [-0.019359100610017776, -0.02121054008603096, 0.03364589810371399], "cos_apical_negg": [-0.019359100610017776, -0.02121054008603096, 0.03364589810371399], "cos_Ac_negg": [-0.019359100610017776, -0.02121054008603096, 0.03364589810371399], "r_norm": [4.916833877563477, 4.846382141113281, 4.838560104370117]}, {"step": 50, "epoch": 0, "train_loss": 0.5560474395751953, "cos_r_negg": [0.43746745586395264, 0.3510897159576416, 0.3023782968521118], "cos_innovation_negg": [0.43746745586395264, 0.3510897159576416, 0.3023782968521118], "cos_apical_negg": [0.43746745586395264, 0.3510897159576416, 0.3023782968521118], "cos_Ac_negg": [0.43746745586395264, 0.3510897159576416, 0.3023782968521118], "r_norm": [1.1692003011703491, 1.1344645023345947, 1.134726881980896]}, {"step": 100, "epoch": 0, "train_loss": 0.20870932936668396, "cos_r_negg": [0.49342286586761475, 0.39893725514411926, 0.3361607789993286], "cos_innovation_negg": [0.49342286586761475, 0.39893725514411926, 0.3361607789993286], "cos_apical_negg": [0.49342286586761475, 0.39893725514411926, 0.3361607789993286], "cos_Ac_negg": [0.49342286586761475, 0.39893725514411926, 0.3361607789993286], "r_norm": [0.9660242795944214, 0.9432772397994995, 0.9440054893493652]}, {"step": 150, "epoch": 0, "train_loss": 0.19792844355106354, "cos_r_negg": [0.4912440776824951, 0.39535844326019287, 0.34642234444618225], "cos_innovation_negg": [0.4912440776824951, 0.39535844326019287, 0.34642234444618225], "cos_apical_negg": [0.4912440776824951, 0.39535844326019287, 0.34642234444618225], "cos_Ac_negg": [0.4912440776824951, 0.39535844326019287, 0.34642234444618225], "r_norm": [0.8132921457290649, 0.7939161658287048, 0.7929287552833557]}, {"step": 200, "epoch": 0, "train_loss": 0.2982172667980194, "cos_r_negg": [0.5103440880775452, 0.4325461983680725, 0.372334361076355], "cos_innovation_negg": [0.5103440880775452, 0.4325461983680725, 0.372334361076355], "cos_apical_negg": [0.5103440880775452, 0.4325461983680725, 0.372334361076355], "cos_Ac_negg": [0.5103440880775452, 0.4325461983680725, 0.372334361076355], "r_norm": [0.7712408304214478, 0.7540644407272339, 0.7526206374168396]}, {"step": 250, "epoch": 0, "train_loss": 0.3501119017601013, "cos_r_negg": [0.5230883359909058, 0.47240060567855835, 0.4013531506061554], "cos_innovation_negg": [0.5230883359909058, 0.47240060567855835, 0.4013531506061554], "cos_apical_negg": [0.5230883359909058, 0.47240060567855835, 0.4013531506061554], "cos_Ac_negg": [0.5230883359909058, 0.47240060567855835, 0.4013531506061554], "r_norm": [0.6660199165344238, 0.6526018977165222, 0.6511543989181519]}, {"step": 300, "epoch": 0, "train_loss": 0.21515600383281708, "cos_r_negg": [0.5010690689086914, 0.46751120686531067, 0.4067574441432953], "cos_innovation_negg": [0.5010690689086914, 0.46751120686531067, 0.4067574441432953], "cos_apical_negg": [0.5010690689086914, 0.46751120686531067, 0.4067574441432953], "cos_Ac_negg": [0.5010690689086914, 0.46751120686531067, 0.4067574441432953], "r_norm": [0.6266919374465942, 0.6157296299934387, 0.6108977198600769]}, {"step": 350, "epoch": 0, "train_loss": 0.2895778715610504, "cos_r_negg": [0.530937671661377, 0.4744516909122467, 0.41647571325302124], "cos_innovation_negg": [0.530937671661377, 0.4744516909122467, 0.41647571325302124], "cos_apical_negg": [0.530937671661377, 0.4744516909122467, 0.41647571325302124], "cos_Ac_negg": [0.530937671661377, 0.4744516909122467, 0.41647571325302124], "r_norm": [0.6396288871765137, 0.629142165184021, 0.6260741949081421]}, {"epoch_end": 0, "step": 391, "test_acc": 0.942, "test_loss": 0.19316934661865234}, {"step": 400, "epoch": 1, "train_loss": 0.17426234483718872, "cos_r_negg": [0.5146183967590332, 0.47954481840133667, 0.4087519645690918], "cos_innovation_negg": [0.5146183967590332, 0.47954481840133667, 0.4087519645690918], "cos_apical_negg": [0.5146183967590332, 0.47954481840133667, 0.4087519645690918], "cos_Ac_negg": [0.5146183967590332, 0.47954481840133667, 0.4087519645690918], "r_norm": [0.5993560552597046, 0.593336284160614, 0.5891167521476746]}, {"step": 450, "epoch": 1, "train_loss": 0.16105760633945465, "cos_r_negg": [0.519124448299408, 0.4896484911441803, 0.42511093616485596], "cos_innovation_negg": [0.519124448299408, 0.4896484911441803, 0.42511093616485596], "cos_apical_negg": [0.519124448299408, 0.4896484911441803, 0.42511093616485596], "cos_Ac_negg": [0.519124448299408, 0.4896484911441803, 0.42511093616485596], "r_norm": [0.5071582794189453, 0.4965616464614868, 0.4943530857563019]}, {"step": 500, "epoch": 1, "train_loss": 0.13914069533348083, "cos_r_negg": [0.5190056562423706, 0.4922347962856293, 0.4461669623851776], "cos_innovation_negg": [0.5190056562423706, 0.4922347962856293, 0.4461669623851776], "cos_apical_negg": [0.5190056562423706, 0.4922347962856293, 0.4461669623851776], "cos_Ac_negg": [0.5190056562423706, 0.4922347962856293, 0.4461669623851776], "r_norm": [0.41323357820510864, 0.4054766595363617, 0.4039217233657837]}, {"step": 550, "epoch": 1, "train_loss": 0.10878869891166687, "cos_r_negg": [0.536530077457428, 0.4964345693588257, 0.45911651849746704], "cos_innovation_negg": [0.536530077457428, 0.4964345693588257, 0.45911651849746704], "cos_apical_negg": [0.536530077457428, 0.4964345693588257, 0.45911651849746704], "cos_Ac_negg": [0.536530077457428, 0.4964345693588257, 0.45911651849746704], "r_norm": [0.46625036001205444, 0.45810452103614807, 0.45437896251678467]}, {"step": 600, "epoch": 1, "train_loss": 0.11212708055973053, "cos_r_negg": [0.5457042455673218, 0.5089675784111023, 0.45809805393218994], "cos_innovation_negg": [0.5457042455673218, 0.5089675784111023, 0.45809805393218994], "cos_apical_negg": [0.5457042455673218, 0.5089675784111023, 0.45809805393218994], "cos_Ac_negg": [0.5457042455673218, 0.5089675784111023, 0.45809805393218994], "r_norm": [0.49606889486312866, 0.48192501068115234, 0.48563116788864136]}, {"step": 650, "epoch": 1, "train_loss": 0.09442958235740662, "cos_r_negg": [0.5230795741081238, 0.4977708160877228, 0.4460787773132324], "cos_innovation_negg": [0.5230795741081238, 0.4977708160877228, 0.4460787773132324], "cos_apical_negg": [0.5230795741081238, 0.4977708160877228, 0.4460787773132324], "cos_Ac_negg": [0.5230795741081238, 0.4977708160877228, 0.4460787773132324], "r_norm": [0.42303162813186646, 0.41450607776641846, 0.41483837366104126]}, {"step": 700, "epoch": 1, "train_loss": 0.14368385076522827, "cos_r_negg": [0.5162389278411865, 0.5024452209472656, 0.43734073638916016], "cos_innovation_negg": [0.5162389278411865, 0.5024452209472656, 0.43734073638916016], "cos_apical_negg": [0.5162389278411865, 0.5024452209472656, 0.43734073638916016], "cos_Ac_negg": [0.5162389278411865, 0.5024452209472656, 0.43734073638916016], "r_norm": [0.4337700307369232, 0.42715704441070557, 0.4249253273010254]}, {"step": 750, "epoch": 1, "train_loss": 0.1789362132549286, "cos_r_negg": [0.5372475385665894, 0.5159597396850586, 0.46373528242111206], "cos_innovation_negg": [0.5372475385665894, 0.5159597396850586, 0.46373528242111206], "cos_apical_negg": [0.5372475385665894, 0.5159597396850586, 0.46373528242111206], "cos_Ac_negg": [0.5372475385665894, 0.5159597396850586, 0.46373528242111206], "r_norm": [0.4579535126686096, 0.4467422068119049, 0.4472985565662384]}, {"epoch_end": 1, "step": 782, "test_acc": 0.9511, "test_loss": 0.17440706024169922}, {"step": 800, "epoch": 2, "train_loss": 0.08245806396007538, "cos_r_negg": [0.5267568826675415, 0.5190484523773193, 0.4569483697414398], "cos_innovation_negg": [0.5267568826675415, 0.5190484523773193, 0.4569483697414398], "cos_apical_negg": [0.5267568826675415, 0.5190484523773193, 0.4569483697414398], "cos_Ac_negg": [0.5267568826675415, 0.5190484523773193, 0.4569483697414398], "r_norm": [0.3940802216529846, 0.38350600004196167, 0.3859628736972809]}, {"step": 850, "epoch": 2, "train_loss": 0.11318741738796234, "cos_r_negg": [0.5415972471237183, 0.5446792840957642, 0.4657430648803711], "cos_innovation_negg": [0.5415972471237183, 0.5446792840957642, 0.4657430648803711], "cos_apical_negg": [0.5415972471237183, 0.5446792840957642, 0.4657430648803711], "cos_Ac_negg": [0.5415972471237183, 0.5446792840957642, 0.4657430648803711], "r_norm": [0.3263637125492096, 0.32023316621780396, 0.32168149948120117]}, {"step": 900, "epoch": 2, "train_loss": 0.06788861006498337, "cos_r_negg": [0.5404047966003418, 0.5453928709030151, 0.4663245975971222], "cos_innovation_negg": [0.5404047966003418, 0.5453928709030151, 0.4663245975971222], "cos_apical_negg": [0.5404047966003418, 0.5453928709030151, 0.4663245975971222], "cos_Ac_negg": [0.5404047966003418, 0.5453928709030151, 0.4663245975971222], "r_norm": [0.4072578549385071, 0.40146565437316895, 0.39743897318840027]}, {"step": 950, "epoch": 2, "train_loss": 0.12236154824495316, "cos_r_negg": [0.5568745136260986, 0.5489026308059692, 0.47848761081695557], "cos_innovation_negg": [0.5568745136260986, 0.5489026308059692, 0.47848761081695557], "cos_apical_negg": [0.5568745136260986, 0.5489026308059692, 0.47848761081695557], "cos_Ac_negg": [0.5568745136260986, 0.5489026308059692, 0.47848761081695557], "r_norm": [0.3096506595611572, 0.30416738986968994, 0.30429601669311523]}, {"step": 1000, "epoch": 2, "train_loss": 0.1695403754711151, "cos_r_negg": [0.5511597394943237, 0.567548394203186, 0.48887598514556885], "cos_innovation_negg": [0.5511597394943237, 0.567548394203186, 0.48887598514556885], "cos_apical_negg": [0.5511597394943237, 0.567548394203186, 0.48887598514556885], "cos_Ac_negg": [0.5511597394943237, 0.567548394203186, 0.48887598514556885], "r_norm": [0.36753782629966736, 0.36040329933166504, 0.3588414192199707]}, {"step": 1050, "epoch": 2, "train_loss": 0.10765781253576279, "cos_r_negg": [0.5588736534118652, 0.5836095809936523, 0.4975379705429077], "cos_innovation_negg": [0.5588736534118652, 0.5836095809936523, 0.4975379705429077], "cos_apical_negg": [0.5588736534118652, 0.5836095809936523, 0.4975379705429077], "cos_Ac_negg": [0.5588736534118652, 0.5836095809936523, 0.4975379705429077], "r_norm": [0.34582000970840454, 0.3401137590408325, 0.33868497610092163]}, {"step": 1100, "epoch": 2, "train_loss": 0.12539835274219513, "cos_r_negg": [0.5508451461791992, 0.5604175329208374, 0.4932308793067932], "cos_innovation_negg": [0.5508451461791992, 0.5604175329208374, 0.4932308793067932], "cos_apical_negg": [0.5508451461791992, 0.5604175329208374, 0.4932308793067932], "cos_Ac_negg": [0.5508451461791992, 0.5604175329208374, 0.4932308793067932], "r_norm": [0.3108765780925751, 0.3024168312549591, 0.3040424883365631]}, {"step": 1150, "epoch": 2, "train_loss": 0.13319088518619537, "cos_r_negg": [0.5785408020019531, 0.580689549446106, 0.493661493062973], "cos_innovation_negg": [0.5785408020019531, 0.580689549446106, 0.493661493062973], "cos_apical_negg": [0.5785408020019531, 0.580689549446106, 0.493661493062973], "cos_Ac_negg": [0.5785408020019531, 0.580689549446106, 0.493661493062973], "r_norm": [0.2865879237651825, 0.2818412780761719, 0.28185901045799255]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9594, "test_loss": 0.1371087953567505}, {"step": 1200, "epoch": 3, "train_loss": 0.07902738451957703, "cos_r_negg": [0.5543697476387024, 0.5753946900367737, 0.49765744805336], "cos_innovation_negg": [0.5543697476387024, 0.5753946900367737, 0.49765744805336], "cos_apical_negg": [0.5543697476387024, 0.5753946900367737, 0.49765744805336], "cos_Ac_negg": [0.5543697476387024, 0.5753946900367737, 0.49765744805336], "r_norm": [0.32674074172973633, 0.32181352376937866, 0.3197745084762573]}, {"step": 1250, "epoch": 3, "train_loss": 0.09491544216871262, "cos_r_negg": [0.5517991781234741, 0.5787758827209473, 0.5108739137649536], "cos_innovation_negg": [0.5517991781234741, 0.5787758827209473, 0.5108739137649536], "cos_apical_negg": [0.5517991781234741, 0.5787758827209473, 0.5108739137649536], "cos_Ac_negg": [0.5517991781234741, 0.5787758827209473, 0.5108739137649536], "r_norm": [0.29178109765052795, 0.28381240367889404, 0.2852678894996643]}, {"step": 1300, "epoch": 3, "train_loss": 0.02431463822722435, "cos_r_negg": [0.5443146228790283, 0.5774863958358765, 0.5164209604263306], "cos_innovation_negg": [0.5443146228790283, 0.5774863958358765, 0.5164209604263306], "cos_apical_negg": [0.5443146228790283, 0.5774863958358765, 0.5164209604263306], "cos_Ac_negg": [0.5443146228790283, 0.5774863958358765, 0.5164209604263306], "r_norm": [0.3149015009403229, 0.30836963653564453, 0.3081222176551819]}, {"step": 1350, "epoch": 3, "train_loss": 0.06450746208429337, "cos_r_negg": [0.5606392621994019, 0.5935269594192505, 0.5256491303443909], "cos_innovation_negg": [0.5606392621994019, 0.5935269594192505, 0.5256491303443909], "cos_apical_negg": [0.5606392621994019, 0.5935269594192505, 0.5256491303443909], "cos_Ac_negg": [0.5606392621994019, 0.5935269594192505, 0.5256491303443909], "r_norm": [0.2790258228778839, 0.2739120125770569, 0.2731637954711914]}, {"step": 1400, "epoch": 3, "train_loss": 0.038685597479343414, "cos_r_negg": [0.5705311298370361, 0.5892266035079956, 0.5223861932754517], "cos_innovation_negg": [0.5705311298370361, 0.5892266035079956, 0.5223861932754517], "cos_apical_negg": [0.5705311298370361, 0.5892266035079956, 0.5223861932754517], "cos_Ac_negg": [0.5705311298370361, 0.5892266035079956, 0.5223861932754517], "r_norm": [0.28946489095687866, 0.2836824953556061, 0.28411585092544556]}, {"step": 1450, "epoch": 3, "train_loss": 0.05291450023651123, "cos_r_negg": [0.5537008047103882, 0.5820235013961792, 0.530899167060852], "cos_innovation_negg": [0.5537008047103882, 0.5820235013961792, 0.530899167060852], "cos_apical_negg": [0.5537008047103882, 0.5820235013961792, 0.530899167060852], "cos_Ac_negg": [0.5537008047103882, 0.5820235013961792, 0.530899167060852], "r_norm": [0.24380138516426086, 0.2381824553012848, 0.24016274511814117]}, {"step": 1500, "epoch": 3, "train_loss": 0.1322896033525467, "cos_r_negg": [0.5590769052505493, 0.5787668228149414, 0.5246911644935608], "cos_innovation_negg": [0.5590769052505493, 0.5787668228149414, 0.5246911644935608], "cos_apical_negg": [0.5590769052505493, 0.5787668228149414, 0.5246911644935608], "cos_Ac_negg": [0.5590769052505493, 0.5787668228149414, 0.5246911644935608], "r_norm": [0.23226672410964966, 0.2268093228340149, 0.22869965434074402]}, {"step": 1550, "epoch": 3, "train_loss": 0.10140091180801392, "cos_r_negg": [0.5600963830947876, 0.5878425240516663, 0.5401417016983032], "cos_innovation_negg": [0.5600963830947876, 0.5878425240516663, 0.5401417016983032], "cos_apical_negg": [0.5600963830947876, 0.5878425240516663, 0.5401417016983032], "cos_Ac_negg": [0.5600963830947876, 0.5878425240516663, 0.5401417016983032], "r_norm": [0.27320921421051025, 0.26625874638557434, 0.26702815294265747]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9632, "test_loss": 0.12679631652832032}, {"step": 1600, "epoch": 4, "train_loss": 0.08276531100273132, "cos_r_negg": [0.5717946887016296, 0.5983544588088989, 0.5367348194122314], "cos_innovation_negg": [0.5717946887016296, 0.5983544588088989, 0.5367348194122314], "cos_apical_negg": [0.5717946887016296, 0.5983544588088989, 0.5367348194122314], "cos_Ac_negg": [0.5717946887016296, 0.5983544588088989, 0.5367348194122314], "r_norm": [0.24837860465049744, 0.2421697974205017, 0.24386495351791382]}, {"step": 1650, "epoch": 4, "train_loss": 0.07333089411258698, "cos_r_negg": [0.5526349544525146, 0.590614914894104, 0.544661283493042], "cos_innovation_negg": [0.5526349544525146, 0.590614914894104, 0.544661283493042], "cos_apical_negg": [0.5526349544525146, 0.590614914894104, 0.544661283493042], "cos_Ac_negg": [0.5526349544525146, 0.590614914894104, 0.544661283493042], "r_norm": [0.24890492856502533, 0.24357996881008148, 0.2439548522233963]}, {"step": 1700, "epoch": 4, "train_loss": 0.06586304306983948, "cos_r_negg": [0.5650207996368408, 0.5971676111221313, 0.545952320098877], "cos_innovation_negg": [0.5650207996368408, 0.5971676111221313, 0.545952320098877], "cos_apical_negg": [0.5650207996368408, 0.5971676111221313, 0.545952320098877], "cos_Ac_negg": [0.5650207996368408, 0.5971676111221313, 0.545952320098877], "r_norm": [0.2353585958480835, 0.22980782389640808, 0.23071803152561188]}, {"step": 1750, "epoch": 4, "train_loss": 0.075168676674366, "cos_r_negg": [0.5610799789428711, 0.5960244536399841, 0.5449928045272827], "cos_innovation_negg": [0.5610799789428711, 0.5960244536399841, 0.5449928045272827], "cos_apical_negg": [0.5610799789428711, 0.5960244536399841, 0.5449928045272827], "cos_Ac_negg": [0.5610799789428711, 0.5960244536399841, 0.5449928045272827], "r_norm": [0.2534013092517853, 0.24448779225349426, 0.24704018235206604]}, {"step": 1800, "epoch": 4, "train_loss": 0.06058744713664055, "cos_r_negg": [0.5512423515319824, 0.5933492183685303, 0.5498859882354736], "cos_innovation_negg": [0.5512423515319824, 0.5933492183685303, 0.5498859882354736], "cos_apical_negg": [0.5512423515319824, 0.5933492183685303, 0.5498859882354736], "cos_Ac_negg": [0.5512423515319824, 0.5933492183685303, 0.5498859882354736], "r_norm": [0.2431870400905609, 0.24097776412963867, 0.23901012539863586]}, {"step": 1850, "epoch": 4, "train_loss": 0.14449454843997955, "cos_r_negg": [0.5511468648910522, 0.5978442430496216, 0.5535270571708679], "cos_innovation_negg": [0.5511468648910522, 0.5978442430496216, 0.5535270571708679], "cos_apical_negg": [0.5511468648910522, 0.5978442430496216, 0.5535270571708679], "cos_Ac_negg": [0.5511468648910522, 0.5978442430496216, 0.5535270571708679], "r_norm": [0.26309657096862793, 0.2573927044868469, 0.2583658695220947]}, {"step": 1900, "epoch": 4, "train_loss": 0.0459626242518425, "cos_r_negg": [0.5704281330108643, 0.6106164455413818, 0.5650182962417603], "cos_innovation_negg": [0.5704281330108643, 0.6106164455413818, 0.5650182962417603], "cos_apical_negg": [0.5704281330108643, 0.6106164455413818, 0.5650182962417603], "cos_Ac_negg": [0.5704281330108643, 0.6106164455413818, 0.5650182962417603], "r_norm": [0.24044054746627808, 0.23593193292617798, 0.23532624542713165]}, {"step": 1950, "epoch": 4, "train_loss": 0.05677869915962219, "cos_r_negg": [0.5598948001861572, 0.6043139696121216, 0.5572119951248169], "cos_innovation_negg": [0.5598948001861572, 0.6043139696121216, 0.5572119951248169], "cos_apical_negg": [0.5598948001861572, 0.6043139696121216, 0.5572119951248169], "cos_Ac_negg": [0.5598948001861572, 0.6043139696121216, 0.5572119951248169], "r_norm": [0.23269245028495789, 0.22904610633850098, 0.228618323802948]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9636, "test_loss": 0.12771956672668458}, {"step": 2000, "epoch": 5, "train_loss": 0.01771124266088009, "cos_r_negg": [0.5826953649520874, 0.6094691753387451, 0.5657365322113037], "cos_innovation_negg": [0.5826953649520874, 0.6094691753387451, 0.5657365322113037], "cos_apical_negg": [0.5826953649520874, 0.6094691753387451, 0.5657365322113037], "cos_Ac_negg": [0.5826953649520874, 0.6094691753387451, 0.5657365322113037], "r_norm": [0.20216906070709229, 0.19860748946666718, 0.19973906874656677]}, {"step": 2050, "epoch": 5, "train_loss": 0.02713404968380928, "cos_r_negg": [0.5666271448135376, 0.607162594795227, 0.5597078204154968], "cos_innovation_negg": [0.5666271448135376, 0.607162594795227, 0.5597078204154968], "cos_apical_negg": [0.5666271448135376, 0.607162594795227, 0.5597078204154968], "cos_Ac_negg": [0.5666271448135376, 0.607162594795227, 0.5597078204154968], "r_norm": [0.1987132430076599, 0.19517919421195984, 0.1967175006866455]}, {"step": 2100, "epoch": 5, "train_loss": 0.017133839428424835, "cos_r_negg": [0.5886457562446594, 0.6221880912780762, 0.5707134008407593], "cos_innovation_negg": [0.5886457562446594, 0.6221880912780762, 0.5707134008407593], "cos_apical_negg": [0.5886457562446594, 0.6221880912780762, 0.5707134008407593], "cos_Ac_negg": [0.5886457562446594, 0.6221880912780762, 0.5707134008407593], "r_norm": [0.21054521203041077, 0.20390814542770386, 0.2054317444562912]}, {"step": 2150, "epoch": 5, "train_loss": 0.005420150235295296, "cos_r_negg": [0.5632246732711792, 0.6007503271102905, 0.5661600828170776], "cos_innovation_negg": [0.5632246732711792, 0.6007503271102905, 0.5661600828170776], "cos_apical_negg": [0.5632246732711792, 0.6007503271102905, 0.5661600828170776], "cos_Ac_negg": [0.5632246732711792, 0.6007503271102905, 0.5661600828170776], "r_norm": [0.20837853848934174, 0.20333851873874664, 0.2044348418712616]}, {"step": 2200, "epoch": 5, "train_loss": 0.017665598541498184, "cos_r_negg": [0.5784574747085571, 0.6043325662612915, 0.5790922045707703], "cos_innovation_negg": [0.5784574747085571, 0.6043325662612915, 0.5790922045707703], "cos_apical_negg": [0.5784574747085571, 0.6043325662612915, 0.5790922045707703], "cos_Ac_negg": [0.5784574747085571, 0.6043325662612915, 0.5790922045707703], "r_norm": [0.2059297114610672, 0.20055487751960754, 0.20234349370002747]}, {"step": 2250, "epoch": 5, "train_loss": 0.03980375453829765, "cos_r_negg": [0.5676448941230774, 0.6039410829544067, 0.5791960954666138], "cos_innovation_negg": [0.5676448941230774, 0.6039410829544067, 0.5791960954666138], "cos_apical_negg": [0.5676448941230774, 0.6039410829544067, 0.5791960954666138], "cos_Ac_negg": [0.5676448941230774, 0.6039410829544067, 0.5791960954666138], "r_norm": [0.20810073614120483, 0.20267745852470398, 0.20394156873226166]}, {"step": 2300, "epoch": 5, "train_loss": 0.04377524182200432, "cos_r_negg": [0.5744448900222778, 0.6133928298950195, 0.5847669839859009], "cos_innovation_negg": [0.5744448900222778, 0.6133928298950195, 0.5847669839859009], "cos_apical_negg": [0.5744448900222778, 0.6133928298950195, 0.5847669839859009], "cos_Ac_negg": [0.5744448900222778, 0.6133928298950195, 0.5847669839859009], "r_norm": [0.16837356984615326, 0.16456538438796997, 0.16513416171073914]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9665, "test_loss": 0.12800794429779053}, {"step": 2350, "epoch": 6, "train_loss": 0.006783369928598404, "cos_r_negg": [0.5734924077987671, 0.6097804307937622, 0.588858425617218], "cos_innovation_negg": [0.5734924077987671, 0.6097804307937622, 0.588858425617218], "cos_apical_negg": [0.5734924077987671, 0.6097804307937622, 0.588858425617218], "cos_Ac_negg": [0.5734924077987671, 0.6097804307937622, 0.588858425617218], "r_norm": [0.18213598430156708, 0.17820268869400024, 0.17873132228851318]}, {"step": 2400, "epoch": 6, "train_loss": 0.029639583081007004, "cos_r_negg": [0.5809173583984375, 0.6193697452545166, 0.6095294952392578], "cos_innovation_negg": [0.5809173583984375, 0.6193697452545166, 0.6095294952392578], "cos_apical_negg": [0.5809173583984375, 0.6193697452545166, 0.6095294952392578], "cos_Ac_negg": [0.5809173583984375, 0.6193697452545166, 0.6095294952392578], "r_norm": [0.19038063287734985, 0.18615368008613586, 0.187724769115448]}, {"step": 2450, "epoch": 6, "train_loss": 0.010544674471020699, "cos_r_negg": [0.5689620971679688, 0.6161101460456848, 0.6044611930847168], "cos_innovation_negg": [0.5689620971679688, 0.6161101460456848, 0.6044611930847168], "cos_apical_negg": [0.5689620971679688, 0.6161101460456848, 0.6044611930847168], "cos_Ac_negg": [0.5689620971679688, 0.6161101460456848, 0.6044611930847168], "r_norm": [0.2135101854801178, 0.209373340010643, 0.21072031557559967]}, {"step": 2500, "epoch": 6, "train_loss": 0.020141305401921272, "cos_r_negg": [0.5781247019767761, 0.6199416518211365, 0.6113945245742798], "cos_innovation_negg": [0.5781247019767761, 0.6199416518211365, 0.6113945245742798], "cos_apical_negg": [0.5781247019767761, 0.6199416518211365, 0.6113945245742798], "cos_Ac_negg": [0.5781247019767761, 0.6199416518211365, 0.6113945245742798], "r_norm": [0.20877400040626526, 0.20529653131961823, 0.20621007680892944]}, {"step": 2550, "epoch": 6, "train_loss": 0.027276888489723206, "cos_r_negg": [0.5745788812637329, 0.6171178221702576, 0.607695460319519], "cos_innovation_negg": [0.5745788812637329, 0.6171178221702576, 0.607695460319519], "cos_apical_negg": [0.5745788812637329, 0.6171178221702576, 0.607695460319519], "cos_Ac_negg": [0.5745788812637329, 0.6171178221702576, 0.607695460319519], "r_norm": [0.22073039412498474, 0.21428953111171722, 0.21619650721549988]}, {"step": 2600, "epoch": 6, "train_loss": 0.03764370456337929, "cos_r_negg": [0.576514482498169, 0.6211690902709961, 0.6162787675857544], "cos_innovation_negg": [0.576514482498169, 0.6211690902709961, 0.6162787675857544], "cos_apical_negg": [0.576514482498169, 0.6211690902709961, 0.6162787675857544], "cos_Ac_negg": [0.576514482498169, 0.6211690902709961, 0.6162787675857544], "r_norm": [0.17158113420009613, 0.1681673377752304, 0.16859003901481628]}, {"step": 2650, "epoch": 6, "train_loss": 0.07222234457731247, "cos_r_negg": [0.5871332883834839, 0.6358972787857056, 0.6140213012695312], "cos_innovation_negg": [0.5871332883834839, 0.6358972787857056, 0.6140213012695312], "cos_apical_negg": [0.5871332883834839, 0.6358972787857056, 0.6140213012695312], "cos_Ac_negg": [0.5871332883834839, 0.6358972787857056, 0.6140213012695312], "r_norm": [0.16949617862701416, 0.16499951481819153, 0.166875422000885]}, {"step": 2700, "epoch": 6, "train_loss": 0.004934626165777445, "cos_r_negg": [0.5820627212524414, 0.6243067979812622, 0.6136757135391235], "cos_innovation_negg": [0.5820627212524414, 0.6243067979812622, 0.6136757135391235], "cos_apical_negg": [0.5820627212524414, 0.6243067979812622, 0.6136757135391235], "cos_Ac_negg": [0.5820627212524414, 0.6243067979812622, 0.6136757135391235], "r_norm": [0.19275519251823425, 0.18920470774173737, 0.19022738933563232]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9675, "test_loss": 0.13033215942382811}, {"step": 2750, "epoch": 7, "train_loss": 0.0007663220749236643, "cos_r_negg": [0.5887814164161682, 0.6255956888198853, 0.6115896105766296], "cos_innovation_negg": [0.5887814164161682, 0.6255956888198853, 0.6115896105766296], "cos_apical_negg": [0.5887814164161682, 0.6255956888198853, 0.6115896105766296], "cos_Ac_negg": [0.5887814164161682, 0.6255956888198853, 0.6115896105766296], "r_norm": [0.1541595160961151, 0.15047265589237213, 0.15321269631385803]}, {"step": 2800, "epoch": 7, "train_loss": 0.007381078787147999, "cos_r_negg": [0.5969140529632568, 0.6320580840110779, 0.623553991317749], "cos_innovation_negg": [0.5969140529632568, 0.6320580840110779, 0.623553991317749], "cos_apical_negg": [0.5969140529632568, 0.6320580840110779, 0.623553991317749], "cos_Ac_negg": [0.5969140529632568, 0.6320580840110779, 0.623553991317749], "r_norm": [0.15880200266838074, 0.15592212975025177, 0.15697628259658813]}, {"step": 2850, "epoch": 7, "train_loss": 0.002672584494575858, "cos_r_negg": [0.5830303430557251, 0.6255103945732117, 0.6196690797805786], "cos_innovation_negg": [0.5830303430557251, 0.6255103945732117, 0.6196690797805786], "cos_apical_negg": [0.5830303430557251, 0.6255103945732117, 0.6196690797805786], "cos_Ac_negg": [0.5830303430557251, 0.6255103945732117, 0.6196690797805786], "r_norm": [0.151914581656456, 0.14865705370903015, 0.15087169408798218]}, {"step": 2900, "epoch": 7, "train_loss": 0.012098554521799088, "cos_r_negg": [0.5773768424987793, 0.623329222202301, 0.6252142190933228], "cos_innovation_negg": [0.5773768424987793, 0.623329222202301, 0.6252142190933228], "cos_apical_negg": [0.5773768424987793, 0.623329222202301, 0.6252142190933228], "cos_Ac_negg": [0.5773768424987793, 0.623329222202301, 0.6252142190933228], "r_norm": [0.15330709517002106, 0.14978253841400146, 0.151128351688385]}, {"step": 2950, "epoch": 7, "train_loss": 0.003346129320561886, "cos_r_negg": [0.5839371681213379, 0.631973385810852, 0.6260716319084167], "cos_innovation_negg": [0.5839371681213379, 0.631973385810852, 0.6260716319084167], "cos_apical_negg": [0.5839371681213379, 0.631973385810852, 0.6260716319084167], "cos_Ac_negg": [0.5839371681213379, 0.631973385810852, 0.6260716319084167], "r_norm": [0.18398334085941315, 0.17977376282215118, 0.1819036602973938]}, {"step": 3000, "epoch": 7, "train_loss": 0.008345598354935646, "cos_r_negg": [0.5800479650497437, 0.6284677386283875, 0.6313223838806152], "cos_innovation_negg": [0.5800479650497437, 0.6284677386283875, 0.6313223838806152], "cos_apical_negg": [0.5800479650497437, 0.6284677386283875, 0.6313223838806152], "cos_Ac_negg": [0.5800479650497437, 0.6284677386283875, 0.6313223838806152], "r_norm": [0.15616059303283691, 0.15302126109600067, 0.15486370027065277]}, {"step": 3050, "epoch": 7, "train_loss": 0.010694846510887146, "cos_r_negg": [0.583375096321106, 0.6364173889160156, 0.6361832022666931], "cos_innovation_negg": [0.583375096321106, 0.6364173889160156, 0.6361832022666931], "cos_apical_negg": [0.583375096321106, 0.6364173889160156, 0.6361832022666931], "cos_Ac_negg": [0.583375096321106, 0.6364173889160156, 0.6361832022666931], "r_norm": [0.16246932744979858, 0.158235102891922, 0.16048797965049744]}, {"step": 3100, "epoch": 7, "train_loss": 0.0018933835672214627, "cos_r_negg": [0.5734519958496094, 0.6262650489807129, 0.636416494846344], "cos_innovation_negg": [0.5734519958496094, 0.6262650489807129, 0.636416494846344], "cos_apical_negg": [0.5734519958496094, 0.6262650489807129, 0.636416494846344], "cos_Ac_negg": [0.5734519958496094, 0.6262650489807129, 0.636416494846344], "r_norm": [0.16744595766067505, 0.16360560059547424, 0.16462942957878113]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.969, "test_loss": 0.14008486652374266}, {"step": 3150, "epoch": 8, "train_loss": 0.01328835915774107, "cos_r_negg": [0.5799121856689453, 0.6348503828048706, 0.6349548101425171], "cos_innovation_negg": [0.5799121856689453, 0.6348503828048706, 0.6349548101425171], "cos_apical_negg": [0.5799121856689453, 0.6348503828048706, 0.6349548101425171], "cos_Ac_negg": [0.5799121856689453, 0.6348503828048706, 0.6349548101425171], "r_norm": [0.16330114006996155, 0.15841633081436157, 0.16031941771507263]}, {"step": 3200, "epoch": 8, "train_loss": 0.025472568348050117, "cos_r_negg": [0.581796407699585, 0.6385154724121094, 0.6402606964111328], "cos_innovation_negg": [0.581796407699585, 0.6385154724121094, 0.6402606964111328], "cos_apical_negg": [0.581796407699585, 0.6385154724121094, 0.6402606964111328], "cos_Ac_negg": [0.581796407699585, 0.6385154724121094, 0.6402606964111328], "r_norm": [0.17319685220718384, 0.16852861642837524, 0.17100343108177185]}, {"step": 3250, "epoch": 8, "train_loss": 0.002426435938104987, "cos_r_negg": [0.5761756896972656, 0.6371506452560425, 0.6444050073623657], "cos_innovation_negg": [0.5761756896972656, 0.6371506452560425, 0.6444050073623657], "cos_apical_negg": [0.5761756896972656, 0.6371506452560425, 0.6444050073623657], "cos_Ac_negg": [0.5761756896972656, 0.6371506452560425, 0.6444050073623657], "r_norm": [0.18577514588832855, 0.1798975020647049, 0.18263466656208038]}, {"step": 3300, "epoch": 8, "train_loss": 0.0063196998089551926, "cos_r_negg": [0.5675809383392334, 0.6289029121398926, 0.6384797096252441], "cos_innovation_negg": [0.5675809383392334, 0.6289029121398926, 0.6384797096252441], "cos_apical_negg": [0.5675809383392334, 0.6289029121398926, 0.6384797096252441], "cos_Ac_negg": [0.5675809383392334, 0.6289029121398926, 0.6384797096252441], "r_norm": [0.16738298535346985, 0.16243907809257507, 0.16465601325035095]}, {"step": 3350, "epoch": 8, "train_loss": 0.005776852369308472, "cos_r_negg": [0.5756241679191589, 0.6365543603897095, 0.646641731262207], "cos_innovation_negg": [0.5756241679191589, 0.6365543603897095, 0.646641731262207], "cos_apical_negg": [0.5756241679191589, 0.6365543603897095, 0.646641731262207], "cos_Ac_negg": [0.5756241679191589, 0.6365543603897095, 0.646641731262207], "r_norm": [0.18019220232963562, 0.17537415027618408, 0.17672228813171387]}, {"step": 3400, "epoch": 8, "train_loss": 0.0017677175346761942, "cos_r_negg": [0.5679070353507996, 0.6268846988677979, 0.6430825591087341], "cos_innovation_negg": [0.5679070353507996, 0.6268846988677979, 0.6430825591087341], "cos_apical_negg": [0.5679070353507996, 0.6268846988677979, 0.6430825591087341], "cos_Ac_negg": [0.5679070353507996, 0.6268846988677979, 0.6430825591087341], "r_norm": [0.16905266046524048, 0.16505110263824463, 0.16670545935630798]}, {"step": 3450, "epoch": 8, "train_loss": 0.019215740263462067, "cos_r_negg": [0.5795387029647827, 0.6390272974967957, 0.6456834077835083], "cos_innovation_negg": [0.5795387029647827, 0.6390272974967957, 0.6456834077835083], "cos_apical_negg": [0.5795387029647827, 0.6390272974967957, 0.6456834077835083], "cos_Ac_negg": [0.5795387029647827, 0.6390272974967957, 0.6456834077835083], "r_norm": [0.17445454001426697, 0.16989797353744507, 0.17152777314186096]}, {"step": 3500, "epoch": 8, "train_loss": 0.01103738322854042, "cos_r_negg": [0.5835047960281372, 0.636309027671814, 0.6513711810112], "cos_innovation_negg": [0.5835047960281372, 0.636309027671814, 0.6513711810112], "cos_apical_negg": [0.5835047960281372, 0.636309027671814, 0.6513711810112], "cos_Ac_negg": [0.5835047960281372, 0.636309027671814, 0.6513711810112], "r_norm": [0.17534372210502625, 0.17152607440948486, 0.17223432660102844]}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9694, "test_loss": 0.13785879878997803}, {"step": 3550, "epoch": 9, "train_loss": 0.002315219258889556, "cos_r_negg": [0.5788799524307251, 0.6343845129013062, 0.6489148139953613], "cos_innovation_negg": [0.5788799524307251, 0.6343845129013062, 0.6489148139953613], "cos_apical_negg": [0.5788799524307251, 0.6343845129013062, 0.6489148139953613], "cos_Ac_negg": [0.5788799524307251, 0.6343845129013062, 0.6489148139953613], "r_norm": [0.16844865679740906, 0.1639837920665741, 0.16495835781097412]}, {"step": 3600, "epoch": 9, "train_loss": 0.001988393720239401, "cos_r_negg": [0.5748826861381531, 0.634721040725708, 0.6506810784339905], "cos_innovation_negg": [0.5748826861381531, 0.634721040725708, 0.6506810784339905], "cos_apical_negg": [0.5748826861381531, 0.634721040725708, 0.6506810784339905], "cos_Ac_negg": [0.5748826861381531, 0.634721040725708, 0.6506810784339905], "r_norm": [0.15075717866420746, 0.14709879457950592, 0.14821848273277283]}, {"step": 3650, "epoch": 9, "train_loss": 0.006571819074451923, "cos_r_negg": [0.5776793956756592, 0.6351725459098816, 0.6550160646438599], "cos_innovation_negg": [0.5776793956756592, 0.6351725459098816, 0.6550160646438599], "cos_apical_negg": [0.5776793956756592, 0.6351725459098816, 0.6550160646438599], "cos_Ac_negg": [0.5776793956756592, 0.6351725459098816, 0.6550160646438599], "r_norm": [0.15181316435337067, 0.14831840991973877, 0.14939305186271667]}, {"step": 3700, "epoch": 9, "train_loss": 0.007451673038303852, "cos_r_negg": [0.5768934488296509, 0.6356926560401917, 0.6547232866287231], "cos_innovation_negg": [0.5768934488296509, 0.6356926560401917, 0.6547232866287231], "cos_apical_negg": [0.5768934488296509, 0.6356926560401917, 0.6547232866287231], "cos_Ac_negg": [0.5768934488296509, 0.6356926560401917, 0.6547232866287231], "r_norm": [0.1481488049030304, 0.1448420286178589, 0.14619898796081543]}, {"step": 3750, "epoch": 9, "train_loss": 0.0056808991357684135, "cos_r_negg": [0.5863108038902283, 0.638932466506958, 0.6598564386367798], "cos_innovation_negg": [0.5863108038902283, 0.638932466506958, 0.6598564386367798], "cos_apical_negg": [0.5863108038902283, 0.638932466506958, 0.6598564386367798], "cos_Ac_negg": [0.5863108038902283, 0.638932466506958, 0.6598564386367798], "r_norm": [0.1510085016489029, 0.14690589904785156, 0.14804449677467346]}, {"step": 3800, "epoch": 9, "train_loss": 0.0014049416640773416, "cos_r_negg": [0.5772558450698853, 0.6375828981399536, 0.6538931131362915], "cos_innovation_negg": [0.5772558450698853, 0.6375828981399536, 0.6538931131362915], "cos_apical_negg": [0.5772558450698853, 0.6375828981399536, 0.6538931131362915], "cos_Ac_negg": [0.5772558450698853, 0.6375828981399536, 0.6538931131362915], "r_norm": [0.1512400209903717, 0.14769288897514343, 0.14897239208221436]}, {"step": 3850, "epoch": 9, "train_loss": 0.0006248891586437821, "cos_r_negg": [0.5762279033660889, 0.6366449594497681, 0.6594094634056091], "cos_innovation_negg": [0.5762279033660889, 0.6366449594497681, 0.6594094634056091], "cos_apical_negg": [0.5762279033660889, 0.6366449594497681, 0.6594094634056091], "cos_Ac_negg": [0.5762279033660889, 0.6366449594497681, 0.6594094634056091], "r_norm": [0.14771828055381775, 0.14383640885353088, 0.14526548981666565]}, {"step": 3900, "epoch": 9, "train_loss": 0.010852477513253689, "cos_r_negg": [0.5803501605987549, 0.6383212804794312, 0.6603769659996033], "cos_innovation_negg": [0.5803501605987549, 0.6383212804794312, 0.6603769659996033], "cos_apical_negg": [0.5803501605987549, 0.6383212804794312, 0.6603769659996033], "cos_Ac_negg": [0.5803501605987549, 0.6383212804794312, 0.6603769659996033], "r_norm": [0.13685648143291473, 0.1331041157245636, 0.13495194911956787]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9699, "test_loss": 0.1425980173110962}, {"step": 3950, "epoch": 10, "train_loss": 0.0011849066941067576, "cos_r_negg": [0.5736705660820007, 0.6352910995483398, 0.659217894077301], "cos_innovation_negg": [0.5736705660820007, 0.6352910995483398, 0.659217894077301], "cos_apical_negg": [0.5736705660820007, 0.6352910995483398, 0.659217894077301], "cos_Ac_negg": [0.5736705660820007, 0.6352910995483398, 0.659217894077301], "r_norm": [0.14788907766342163, 0.14373983442783356, 0.14589370787143707]}, {"step": 4000, "epoch": 10, "train_loss": 0.0009897679556161165, "cos_r_negg": [0.575480580329895, 0.6365771293640137, 0.6638668775558472], "cos_innovation_negg": [0.575480580329895, 0.6365771293640137, 0.6638668775558472], "cos_apical_negg": [0.575480580329895, 0.6365771293640137, 0.6638668775558472], "cos_Ac_negg": [0.575480580329895, 0.6365771293640137, 0.6638668775558472], "r_norm": [0.15530265867710114, 0.15087273716926575, 0.15281391143798828]}, {"step": 4050, "epoch": 10, "train_loss": 0.0009412005310878158, "cos_r_negg": [0.572576642036438, 0.6350282430648804, 0.6612375974655151], "cos_innovation_negg": [0.572576642036438, 0.6350282430648804, 0.6612375974655151], "cos_apical_negg": [0.572576642036438, 0.6350282430648804, 0.6612375974655151], "cos_Ac_negg": [0.572576642036438, 0.6350282430648804, 0.6612375974655151], "r_norm": [0.14468851685523987, 0.1412726640701294, 0.1432020664215088]}, {"step": 4100, "epoch": 10, "train_loss": 0.0013592280447483063, "cos_r_negg": [0.5738518238067627, 0.6351974010467529, 0.6628438234329224], "cos_innovation_negg": [0.5738518238067627, 0.6351974010467529, 0.6628438234329224], "cos_apical_negg": [0.5738518238067627, 0.6351974010467529, 0.6628438234329224], "cos_Ac_negg": [0.5738518238067627, 0.6351974010467529, 0.6628438234329224], "r_norm": [0.16022688150405884, 0.15651826560497284, 0.15799832344055176]}, {"step": 4150, "epoch": 10, "train_loss": 0.0002184188924729824, "cos_r_negg": [0.5791216492652893, 0.6367849707603455, 0.6641585826873779], "cos_innovation_negg": [0.5791216492652893, 0.6367849707603455, 0.6641585826873779], "cos_apical_negg": [0.5791216492652893, 0.6367849707603455, 0.6641585826873779], "cos_Ac_negg": [0.5791216492652893, 0.6367849707603455, 0.6641585826873779], "r_norm": [0.14311376214027405, 0.14003947377204895, 0.14143389463424683]}, {"step": 4200, "epoch": 10, "train_loss": 0.0009019998833537102, "cos_r_negg": [0.5760563611984253, 0.6389188170433044, 0.6654485464096069], "cos_innovation_negg": [0.5760563611984253, 0.6389188170433044, 0.6654485464096069], "cos_apical_negg": [0.5760563611984253, 0.6389188170433044, 0.6654485464096069], "cos_Ac_negg": [0.5760563611984253, 0.6389188170433044, 0.6654485464096069], "r_norm": [0.13621452450752258, 0.13339275121688843, 0.1347586065530777]}, {"step": 4250, "epoch": 10, "train_loss": 0.002545118797570467, "cos_r_negg": [0.5764381885528564, 0.636925458908081, 0.6655938625335693], "cos_innovation_negg": [0.5764381885528564, 0.636925458908081, 0.6655938625335693], "cos_apical_negg": [0.5764381885528564, 0.636925458908081, 0.6655938625335693], "cos_Ac_negg": [0.5764381885528564, 0.636925458908081, 0.6655938625335693], "r_norm": [0.14012610912322998, 0.13696923851966858, 0.13868092000484467]}, {"step": 4300, "epoch": 10, "train_loss": 0.0007184909773059189, "cos_r_negg": [0.5714229345321655, 0.6362907886505127, 0.6645750999450684], "cos_innovation_negg": [0.5714229345321655, 0.6362907886505127, 0.6645750999450684], "cos_apical_negg": [0.5714229345321655, 0.6362907886505127, 0.6645750999450684], "cos_Ac_negg": [0.5714229345321655, 0.6362907886505127, 0.6645750999450684], "r_norm": [0.1335465908050537, 0.13064071536064148, 0.13241830468177795]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9692, "test_loss": 0.149364381980896}, {"step": 4350, "epoch": 11, "train_loss": 0.001088172197341919, "cos_r_negg": [0.5710287094116211, 0.6341713070869446, 0.6641830205917358], "cos_innovation_negg": [0.5710287094116211, 0.6341713070869446, 0.6641830205917358], "cos_apical_negg": [0.5710287094116211, 0.6341713070869446, 0.6641830205917358], "cos_Ac_negg": [0.5710287094116211, 0.6341713070869446, 0.6641830205917358], "r_norm": [0.1298397332429886, 0.12719902396202087, 0.12864363193511963]}, {"step": 4400, "epoch": 11, "train_loss": 0.0004912855802103877, "cos_r_negg": [0.5738987922668457, 0.6349611282348633, 0.6645448207855225], "cos_innovation_negg": [0.5738987922668457, 0.6349611282348633, 0.6645448207855225], "cos_apical_negg": [0.5738987922668457, 0.6349611282348633, 0.6645448207855225], "cos_Ac_negg": [0.5738987922668457, 0.6349611282348633, 0.6645448207855225], "r_norm": [0.12954868376255035, 0.12682847678661346, 0.12830987572669983]}, {"step": 4450, "epoch": 11, "train_loss": 0.005249932408332825, "cos_r_negg": [0.5733698606491089, 0.6392396688461304, 0.6676554679870605], "cos_innovation_negg": [0.5733698606491089, 0.6392396688461304, 0.6676554679870605], "cos_apical_negg": [0.5733698606491089, 0.6392396688461304, 0.6676554679870605], "cos_Ac_negg": [0.5733698606491089, 0.6392396688461304, 0.6676554679870605], "r_norm": [0.13459138572216034, 0.13149142265319824, 0.13286541402339935]}, {"step": 4500, "epoch": 11, "train_loss": 9.150458208750933e-05, "cos_r_negg": [0.5693934559822083, 0.6387389302253723, 0.6674418449401855], "cos_innovation_negg": [0.5693934559822083, 0.6387389302253723, 0.6674418449401855], "cos_apical_negg": [0.5693934559822083, 0.6387389302253723, 0.6674418449401855], "cos_Ac_negg": [0.5693934559822083, 0.6387389302253723, 0.6674418449401855], "r_norm": [0.13178330659866333, 0.12891516089439392, 0.1303371787071228]}, {"step": 4550, "epoch": 11, "train_loss": 0.0027633458375930786, "cos_r_negg": [0.5651710033416748, 0.6335588693618774, 0.6671782732009888], "cos_innovation_negg": [0.5651710033416748, 0.6335588693618774, 0.6671782732009888], "cos_apical_negg": [0.5651710033416748, 0.6335588693618774, 0.6671782732009888], "cos_Ac_negg": [0.5651710033416748, 0.6335588693618774, 0.6671782732009888], "r_norm": [0.14456260204315186, 0.1411486566066742, 0.1426524817943573]}, {"step": 4600, "epoch": 11, "train_loss": 0.0002579913125373423, "cos_r_negg": [0.5687362551689148, 0.636699914932251, 0.6704668998718262], "cos_innovation_negg": [0.5687362551689148, 0.636699914932251, 0.6704668998718262], "cos_apical_negg": [0.5687362551689148, 0.636699914932251, 0.6704668998718262], "cos_Ac_negg": [0.5687362551689148, 0.636699914932251, 0.6704668998718262], "r_norm": [0.14014819264411926, 0.13707773387432098, 0.138213649392128]}, {"step": 4650, "epoch": 11, "train_loss": 0.00035353790735825896, "cos_r_negg": [0.5665009617805481, 0.633368194103241, 0.669823408126831], "cos_innovation_negg": [0.5665009617805481, 0.633368194103241, 0.669823408126831], "cos_apical_negg": [0.5665009617805481, 0.633368194103241, 0.669823408126831], "cos_Ac_negg": [0.5665009617805481, 0.633368194103241, 0.669823408126831], "r_norm": [0.1399710774421692, 0.13727745413780212, 0.13824251294136047]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9703, "test_loss": 0.1514739902496338}, {"step": 4700, "epoch": 12, "train_loss": 0.0016259646508842707, "cos_r_negg": [0.5645482540130615, 0.6323161125183105, 0.6685330867767334], "cos_innovation_negg": [0.5645482540130615, 0.6323161125183105, 0.6685330867767334], "cos_apical_negg": [0.5645482540130615, 0.6323161125183105, 0.6685330867767334], "cos_Ac_negg": [0.5645482540130615, 0.6323161125183105, 0.6685330867767334], "r_norm": [0.13492870330810547, 0.1323074996471405, 0.13351865112781525]}, {"step": 4750, "epoch": 12, "train_loss": 0.0007716660620644689, "cos_r_negg": [0.5670640468597412, 0.636359453201294, 0.671646773815155], "cos_innovation_negg": [0.5670640468597412, 0.636359453201294, 0.671646773815155], "cos_apical_negg": [0.5670640468597412, 0.636359453201294, 0.671646773815155], "cos_Ac_negg": [0.5670640468597412, 0.636359453201294, 0.671646773815155], "r_norm": [0.13707339763641357, 0.13417401909828186, 0.1354566216468811]}, {"step": 4800, "epoch": 12, "train_loss": 0.00043174016172997653, "cos_r_negg": [0.5645847320556641, 0.6352803707122803, 0.6712182760238647], "cos_innovation_negg": [0.5645847320556641, 0.6352803707122803, 0.6712182760238647], "cos_apical_negg": [0.5645847320556641, 0.6352803707122803, 0.6712182760238647], "cos_Ac_negg": [0.5645847320556641, 0.6352803707122803, 0.6712182760238647], "r_norm": [0.13404728472232819, 0.1311521828174591, 0.13259340822696686]}, {"step": 4850, "epoch": 12, "train_loss": 0.0005846355343237519, "cos_r_negg": [0.5658779144287109, 0.6339683532714844, 0.6700609922409058], "cos_innovation_negg": [0.5658779144287109, 0.6339683532714844, 0.6700609922409058], "cos_apical_negg": [0.5658779144287109, 0.6339683532714844, 0.6700609922409058], "cos_Ac_negg": [0.5658779144287109, 0.6339683532714844, 0.6700609922409058], "r_norm": [0.1345229148864746, 0.13165411353111267, 0.13307037949562073]}, {"step": 4900, "epoch": 12, "train_loss": 0.0010823255870491266, "cos_r_negg": [0.5680403709411621, 0.6369401812553406, 0.6722495555877686], "cos_innovation_negg": [0.5680403709411621, 0.6369401812553406, 0.6722495555877686], "cos_apical_negg": [0.5680403709411621, 0.6369401812553406, 0.6722495555877686], "cos_Ac_negg": [0.5680403709411621, 0.6369401812553406, 0.6722495555877686], "r_norm": [0.1368342787027359, 0.13402937352657318, 0.13532841205596924]}, {"step": 4950, "epoch": 12, "train_loss": 0.00016942250658757985, "cos_r_negg": [0.5621670484542847, 0.632524311542511, 0.670006275177002], "cos_innovation_negg": [0.5621670484542847, 0.632524311542511, 0.670006275177002], "cos_apical_negg": [0.5621670484542847, 0.632524311542511, 0.670006275177002], "cos_Ac_negg": [0.5621670484542847, 0.632524311542511, 0.670006275177002], "r_norm": [0.13597753643989563, 0.13311836123466492, 0.13452139496803284]}, {"step": 5000, "epoch": 12, "train_loss": 0.000977246672846377, "cos_r_negg": [0.5662513971328735, 0.6344373822212219, 0.6723304390907288], "cos_innovation_negg": [0.5662513971328735, 0.6344373822212219, 0.6723304390907288], "cos_apical_negg": [0.5662513971328735, 0.6344373822212219, 0.6723304390907288], "cos_Ac_negg": [0.5662513971328735, 0.6344373822212219, 0.6723304390907288], "r_norm": [0.14220905303955078, 0.13893869519233704, 0.1404062807559967]}, {"step": 5050, "epoch": 12, "train_loss": 0.00040352315409108996, "cos_r_negg": [0.5651079416275024, 0.6322027444839478, 0.6704139113426208], "cos_innovation_negg": [0.5651079416275024, 0.6322027444839478, 0.6704139113426208], "cos_apical_negg": [0.5651079416275024, 0.6322027444839478, 0.6704139113426208], "cos_Ac_negg": [0.5651079416275024, 0.6322027444839478, 0.6704139113426208], "r_norm": [0.14014022052288055, 0.13726016879081726, 0.13854330778121948]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9708, "test_loss": 0.1528864547729492}, {"step": 5100, "epoch": 13, "train_loss": 0.0008149973000399768, "cos_r_negg": [0.5646706819534302, 0.633665144443512, 0.6713343262672424], "cos_innovation_negg": [0.5646706819534302, 0.633665144443512, 0.6713343262672424], "cos_apical_negg": [0.5646706819534302, 0.633665144443512, 0.6713343262672424], "cos_Ac_negg": [0.5646706819534302, 0.633665144443512, 0.6713343262672424], "r_norm": [0.13733923435211182, 0.13432034850120544, 0.1357450783252716]}, {"step": 5150, "epoch": 13, "train_loss": 0.0007546095876023173, "cos_r_negg": [0.5666807889938354, 0.6334596276283264, 0.6712764501571655], "cos_innovation_negg": [0.5666807889938354, 0.6334596276283264, 0.6712764501571655], "cos_apical_negg": [0.5666807889938354, 0.6334596276283264, 0.6712764501571655], "cos_Ac_negg": [0.5666807889938354, 0.6334596276283264, 0.6712764501571655], "r_norm": [0.134781613945961, 0.13192489743232727, 0.13326333463191986]}, {"step": 5200, "epoch": 13, "train_loss": 0.0005536735407076776, "cos_r_negg": [0.566348671913147, 0.6340305805206299, 0.6721346974372864], "cos_innovation_negg": [0.566348671913147, 0.6340305805206299, 0.6721346974372864], "cos_apical_negg": [0.566348671913147, 0.6340305805206299, 0.6721346974372864], "cos_Ac_negg": [0.566348671913147, 0.6340305805206299, 0.6721346974372864], "r_norm": [0.13521593809127808, 0.1322522610425949, 0.13364171981811523]}, {"step": 5250, "epoch": 13, "train_loss": 0.00010934581223409623, "cos_r_negg": [0.5649901628494263, 0.6342901587486267, 0.6720361709594727], "cos_innovation_negg": [0.5649901628494263, 0.6342901587486267, 0.6720361709594727], "cos_apical_negg": [0.5649901628494263, 0.6342901587486267, 0.6720361709594727], "cos_Ac_negg": [0.5649901628494263, 0.6342901587486267, 0.6720361709594727], "r_norm": [0.13634179532527924, 0.13348135352134705, 0.13484670221805573]}, {"step": 5300, "epoch": 13, "train_loss": 0.0009097770671360195, "cos_r_negg": [0.5674624443054199, 0.6369792222976685, 0.6742740869522095], "cos_innovation_negg": [0.5674624443054199, 0.6369792222976685, 0.6742740869522095], "cos_apical_negg": [0.5674624443054199, 0.6369792222976685, 0.6742740869522095], "cos_Ac_negg": [0.5674624443054199, 0.6369792222976685, 0.6742740869522095], "r_norm": [0.13878759741783142, 0.13579849898815155, 0.13710427284240723]}, {"step": 5350, "epoch": 13, "train_loss": 0.0009050553780980408, "cos_r_negg": [0.5653352737426758, 0.6374245882034302, 0.6754180192947388], "cos_innovation_negg": [0.5653352737426758, 0.6374245882034302, 0.6754180192947388], "cos_apical_negg": [0.5653352737426758, 0.6374245882034302, 0.6754180192947388], "cos_Ac_negg": [0.5653352737426758, 0.6374245882034302, 0.6754180192947388], "r_norm": [0.143208846449852, 0.13980285823345184, 0.14147929847240448]}, {"step": 5400, "epoch": 13, "train_loss": 0.001244271406903863, "cos_r_negg": [0.5648072361946106, 0.6343165636062622, 0.6721306443214417], "cos_innovation_negg": [0.5648072361946106, 0.6343165636062622, 0.6721306443214417], "cos_apical_negg": [0.5648072361946106, 0.6343165636062622, 0.6721306443214417], "cos_Ac_negg": [0.5648072361946106, 0.6343165636062622, 0.6721306443214417], "r_norm": [0.1378374993801117, 0.13487277925014496, 0.13645434379577637]}, {"step": 5450, "epoch": 13, "train_loss": 0.0003554274735506624, "cos_r_negg": [0.5650583505630493, 0.6358937621116638, 0.6737478375434875], "cos_innovation_negg": [0.5650583505630493, 0.6358937621116638, 0.6737478375434875], "cos_apical_negg": [0.5650583505630493, 0.6358937621116638, 0.6737478375434875], "cos_Ac_negg": [0.5650583505630493, 0.6358937621116638, 0.6737478375434875], "r_norm": [0.1395883560180664, 0.13642869889736176, 0.13794764876365662]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9703, "test_loss": 0.15620655040740966}, {"step": 5500, "epoch": 14, "train_loss": 0.00014389192801900208, "cos_r_negg": [0.5661411285400391, 0.6336849927902222, 0.6735123991966248], "cos_innovation_negg": [0.5661411285400391, 0.6336849927902222, 0.6735123991966248], "cos_apical_negg": [0.5661411285400391, 0.6336849927902222, 0.6735123991966248], "cos_Ac_negg": [0.5661411285400391, 0.6336849927902222, 0.6735123991966248], "r_norm": [0.13966697454452515, 0.13667698204517365, 0.1379842311143875]}, {"step": 5550, "epoch": 14, "train_loss": 0.00032514159101992846, "cos_r_negg": [0.568307638168335, 0.6357816457748413, 0.6740419864654541], "cos_innovation_negg": [0.568307638168335, 0.6357816457748413, 0.6740419864654541], "cos_apical_negg": [0.568307638168335, 0.6357816457748413, 0.6740419864654541], "cos_Ac_negg": [0.568307638168335, 0.6357816457748413, 0.6740419864654541], "r_norm": [0.13687950372695923, 0.13394951820373535, 0.13538441061973572]}, {"step": 5600, "epoch": 14, "train_loss": 0.0007345066987909377, "cos_r_negg": [0.5671765804290771, 0.6357535123825073, 0.6742692589759827], "cos_innovation_negg": [0.5671765804290771, 0.6357535123825073, 0.6742692589759827], "cos_apical_negg": [0.5671765804290771, 0.6357535123825073, 0.6742692589759827], "cos_Ac_negg": [0.5671765804290771, 0.6357535123825073, 0.6742692589759827], "r_norm": [0.13659030199050903, 0.13359612226486206, 0.13506419956684113]}, {"step": 5650, "epoch": 14, "train_loss": 0.00044516552588902414, "cos_r_negg": [0.5668216943740845, 0.6360499858856201, 0.6737312078475952], "cos_innovation_negg": [0.5668216943740845, 0.6360499858856201, 0.6737312078475952], "cos_apical_negg": [0.5668216943740845, 0.6360499858856201, 0.6737312078475952], "cos_Ac_negg": [0.5668216943740845, 0.6360499858856201, 0.6737312078475952], "r_norm": [0.13384005427360535, 0.1310320496559143, 0.13252440094947815]}, {"step": 5700, "epoch": 14, "train_loss": 0.00030808517476543784, "cos_r_negg": [0.5646768808364868, 0.634541392326355, 0.6743097305297852], "cos_innovation_negg": [0.5646768808364868, 0.634541392326355, 0.6743097305297852], "cos_apical_negg": [0.5646768808364868, 0.634541392326355, 0.6743097305297852], "cos_Ac_negg": [0.5646768808364868, 0.634541392326355, 0.6743097305297852], "r_norm": [0.13691291213035583, 0.134189635515213, 0.1354805827140808]}, {"step": 5750, "epoch": 14, "train_loss": 0.0008174413815140724, "cos_r_negg": [0.5676316022872925, 0.6378339529037476, 0.6756142377853394], "cos_innovation_negg": [0.5676316022872925, 0.6378339529037476, 0.6756142377853394], "cos_apical_negg": [0.5676316022872925, 0.6378339529037476, 0.6756142377853394], "cos_Ac_negg": [0.5676316022872925, 0.6378339529037476, 0.6756142377853394], "r_norm": [0.1382167637348175, 0.13516280055046082, 0.13677388429641724]}, {"step": 5800, "epoch": 14, "train_loss": 0.0019093691371381283, "cos_r_negg": [0.5625150203704834, 0.6335671544075012, 0.6739863157272339], "cos_innovation_negg": [0.5625150203704834, 0.6335671544075012, 0.6739863157272339], "cos_apical_negg": [0.5625150203704834, 0.6335671544075012, 0.6739863157272339], "cos_Ac_negg": [0.5625150203704834, 0.6335671544075012, 0.6739863157272339], "r_norm": [0.13615825772285461, 0.13332857191562653, 0.13472163677215576]}, {"step": 5850, "epoch": 14, "train_loss": 0.0007232567295432091, "cos_r_negg": [0.5634725093841553, 0.6336246728897095, 0.6747287511825562], "cos_innovation_negg": [0.5634725093841553, 0.6336246728897095, 0.6747287511825562], "cos_apical_negg": [0.5634725093841553, 0.6336246728897095, 0.6747287511825562], "cos_Ac_negg": [0.5634725093841553, 0.6336246728897095, 0.6747287511825562], "r_norm": [0.13549679517745972, 0.13267962634563446, 0.13405629992485046]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9706, "test_loss": 0.15855002174377442}, {"step": 5900, "epoch": 15, "train_loss": 0.0004236867243889719, "cos_r_negg": [0.5641131401062012, 0.6351121068000793, 0.6758198738098145], "cos_innovation_negg": [0.5641131401062012, 0.6351121068000793, 0.6758198738098145], "cos_apical_negg": [0.5641131401062012, 0.6351121068000793, 0.6758198738098145], "cos_Ac_negg": [0.5641131401062012, 0.6351121068000793, 0.6758198738098145], "r_norm": [0.13779672980308533, 0.13477648794651031, 0.1363215297460556]}, {"step": 5950, "epoch": 15, "train_loss": 0.0006461036391556263, "cos_r_negg": [0.5634825229644775, 0.6356934309005737, 0.6758770942687988], "cos_innovation_negg": [0.5634825229644775, 0.6356934309005737, 0.6758770942687988], "cos_apical_negg": [0.5634825229644775, 0.6356934309005737, 0.6758770942687988], "cos_Ac_negg": [0.5634825229644775, 0.6356934309005737, 0.6758770942687988], "r_norm": [0.13786721229553223, 0.13480442762374878, 0.1363658905029297]}, {"step": 6000, "epoch": 15, "train_loss": 0.0005086037563160062, "cos_r_negg": [0.5633976459503174, 0.6348764300346375, 0.6754307746887207], "cos_innovation_negg": [0.5633976459503174, 0.6348764300346375, 0.6754307746887207], "cos_apical_negg": [0.5633976459503174, 0.6348764300346375, 0.6754307746887207], "cos_Ac_negg": [0.5633976459503174, 0.6348764300346375, 0.6754307746887207], "r_norm": [0.13768622279167175, 0.13461238145828247, 0.13614992797374725]}, {"step": 6050, "epoch": 15, "train_loss": 0.0019304364686831832, "cos_r_negg": [0.563418447971344, 0.6360549926757812, 0.6759344935417175], "cos_innovation_negg": [0.563418447971344, 0.6360549926757812, 0.6759344935417175], "cos_apical_negg": [0.563418447971344, 0.6360549926757812, 0.6759344935417175], "cos_Ac_negg": [0.563418447971344, 0.6360549926757812, 0.6759344935417175], "r_norm": [0.13924461603164673, 0.1359919309616089, 0.13774970173835754]}, {"step": 6100, "epoch": 15, "train_loss": 0.00045523588778451085, "cos_r_negg": [0.5637054443359375, 0.6348055601119995, 0.6754481792449951], "cos_innovation_negg": [0.5637054443359375, 0.6348055601119995, 0.6754481792449951], "cos_apical_negg": [0.5637054443359375, 0.6348055601119995, 0.6754481792449951], "cos_Ac_negg": [0.5637054443359375, 0.6348055601119995, 0.6754481792449951], "r_norm": [0.13969182968139648, 0.136508509516716, 0.1381172239780426]}, {"step": 6150, "epoch": 15, "train_loss": 0.00024037175171542913, "cos_r_negg": [0.5640235543251038, 0.6353939771652222, 0.674989640712738], "cos_innovation_negg": [0.5640235543251038, 0.6353939771652222, 0.674989640712738], "cos_apical_negg": [0.5640235543251038, 0.6353939771652222, 0.674989640712738], "cos_Ac_negg": [0.5640235543251038, 0.6353939771652222, 0.674989640712738], "r_norm": [0.1370169222354889, 0.1340174674987793, 0.1354735791683197]}, {"step": 6200, "epoch": 15, "train_loss": 0.00033100228756666183, "cos_r_negg": [0.5642508268356323, 0.6354169845581055, 0.6757903695106506], "cos_innovation_negg": [0.5642508268356323, 0.6354169845581055, 0.6757903695106506], "cos_apical_negg": [0.5642508268356323, 0.6354169845581055, 0.6757903695106506], "cos_Ac_negg": [0.5642508268356323, 0.6354169845581055, 0.6757903695106506], "r_norm": [0.1374688446521759, 0.13445121049880981, 0.13596408069133759]}, {"step": 6250, "epoch": 15, "train_loss": 0.00046477728756144643, "cos_r_negg": [0.563748300075531, 0.6357822418212891, 0.6759568452835083], "cos_innovation_negg": [0.563748300075531, 0.6357822418212891, 0.6759568452835083], "cos_apical_negg": [0.563748300075531, 0.6357822418212891, 0.6759568452835083], "cos_Ac_negg": [0.563748300075531, 0.6357822418212891, 0.6759568452835083], "r_norm": [0.13629740476608276, 0.1332845687866211, 0.1348167359828949]}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9707, "test_loss": 0.1609266523361206}, {"step": 6300, "epoch": 16, "train_loss": 0.0003884444013237953, "cos_r_negg": [0.5619405508041382, 0.6350917816162109, 0.6756013631820679], "cos_innovation_negg": [0.5619405508041382, 0.6350917816162109, 0.6756013631820679], "cos_apical_negg": [0.5619405508041382, 0.6350917816162109, 0.6756013631820679], "cos_Ac_negg": [0.5619405508041382, 0.6350917816162109, 0.6756013631820679], "r_norm": [0.13716721534729004, 0.1340622454881668, 0.1355431228876114]}, {"step": 6350, "epoch": 16, "train_loss": 0.00035933026811107993, "cos_r_negg": [0.5642749667167664, 0.6346299648284912, 0.6761182546615601], "cos_innovation_negg": [0.5642749667167664, 0.6346299648284912, 0.6761182546615601], "cos_apical_negg": [0.5642749667167664, 0.6346299648284912, 0.6761182546615601], "cos_Ac_negg": [0.5642749667167664, 0.6346299648284912, 0.6761182546615601], "r_norm": [0.1398780345916748, 0.1365678608417511, 0.13819582760334015]}, {"step": 6400, "epoch": 16, "train_loss": 0.0009485327173024416, "cos_r_negg": [0.5635079145431519, 0.6347522735595703, 0.6752529144287109], "cos_innovation_negg": [0.5635079145431519, 0.6347522735595703, 0.6752529144287109], "cos_apical_negg": [0.5635079145431519, 0.6347522735595703, 0.6752529144287109], "cos_Ac_negg": [0.5635079145431519, 0.6347522735595703, 0.6752529144287109], "r_norm": [0.136736661195755, 0.1335957646369934, 0.13517630100250244]}, {"step": 6450, "epoch": 16, "train_loss": 0.0002637112629599869, "cos_r_negg": [0.5610992908477783, 0.6337746977806091, 0.6755675077438354], "cos_innovation_negg": [0.5610992908477783, 0.6337746977806091, 0.6755675077438354], "cos_apical_negg": [0.5610992908477783, 0.6337746977806091, 0.6755675077438354], "cos_Ac_negg": [0.5610992908477783, 0.6337746977806091, 0.6755675077438354], "r_norm": [0.1371588110923767, 0.13407710194587708, 0.13566350936889648]}, {"step": 6500, "epoch": 16, "train_loss": 0.0002781759831123054, "cos_r_negg": [0.5615055561065674, 0.6338765621185303, 0.675254225730896], "cos_innovation_negg": [0.5615055561065674, 0.6338765621185303, 0.675254225730896], "cos_apical_negg": [0.5615055561065674, 0.6338765621185303, 0.675254225730896], "cos_Ac_negg": [0.5615055561065674, 0.6338765621185303, 0.675254225730896], "r_norm": [0.13475577533245087, 0.13185355067253113, 0.1333393156528473]}, {"step": 6550, "epoch": 16, "train_loss": 0.00024034807574935257, "cos_r_negg": [0.5622308254241943, 0.6329660415649414, 0.6753203868865967], "cos_innovation_negg": [0.5622308254241943, 0.6329660415649414, 0.6753203868865967], "cos_apical_negg": [0.5622308254241943, 0.6329660415649414, 0.6753203868865967], "cos_Ac_negg": [0.5622308254241943, 0.6329660415649414, 0.6753203868865967], "r_norm": [0.1379355490207672, 0.13485538959503174, 0.1364414095878601]}, {"step": 6600, "epoch": 16, "train_loss": 0.0003139325126539916, "cos_r_negg": [0.5620595812797546, 0.6342012882232666, 0.676263689994812], "cos_innovation_negg": [0.5620595812797546, 0.6342012882232666, 0.676263689994812], "cos_apical_negg": [0.5620595812797546, 0.6342012882232666, 0.676263689994812], "cos_Ac_negg": [0.5620595812797546, 0.6342012882232666, 0.676263689994812], "r_norm": [0.13755595684051514, 0.13450521230697632, 0.13612490892410278]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9706, "test_loss": 0.1633720708847046}, {"step": 6650, "epoch": 17, "train_loss": 0.00038289488293230534, "cos_r_negg": [0.5615799427032471, 0.6341778039932251, 0.6761041879653931], "cos_innovation_negg": [0.5615799427032471, 0.6341778039932251, 0.6761041879653931], "cos_apical_negg": [0.5615799427032471, 0.6341778039932251, 0.6761041879653931], "cos_Ac_negg": [0.5615799427032471, 0.6341778039932251, 0.6761041879653931], "r_norm": [0.13698457181453705, 0.1339055597782135, 0.135618656873703]}, {"step": 6700, "epoch": 17, "train_loss": 0.0003385598538443446, "cos_r_negg": [0.5621045827865601, 0.6340005397796631, 0.6771783828735352], "cos_innovation_negg": [0.5621045827865601, 0.6340005397796631, 0.6771783828735352], "cos_apical_negg": [0.5621045827865601, 0.6340005397796631, 0.6771783828735352], "cos_Ac_negg": [0.5621045827865601, 0.6340005397796631, 0.6771783828735352], "r_norm": [0.13915733993053436, 0.13599219918251038, 0.1376335471868515]}, {"step": 6750, "epoch": 17, "train_loss": 0.0005925591103732586, "cos_r_negg": [0.5633838176727295, 0.6342360973358154, 0.6774816513061523], "cos_innovation_negg": [0.5633838176727295, 0.6342360973358154, 0.6774816513061523], "cos_apical_negg": [0.5633838176727295, 0.6342360973358154, 0.6774816513061523], "cos_Ac_negg": [0.5633838176727295, 0.6342360973358154, 0.6774816513061523], "r_norm": [0.1394316852092743, 0.1362670660018921, 0.13781385123729706]}, {"step": 6800, "epoch": 17, "train_loss": 0.0005954133812338114, "cos_r_negg": [0.5623114109039307, 0.634290337562561, 0.6767703294754028], "cos_innovation_negg": [0.5623114109039307, 0.634290337562561, 0.6767703294754028], "cos_apical_negg": [0.5623114109039307, 0.634290337562561, 0.6767703294754028], "cos_Ac_negg": [0.5623114109039307, 0.634290337562561, 0.6767703294754028], "r_norm": [0.13722273707389832, 0.134128600358963, 0.1357390582561493]}, {"step": 6850, "epoch": 17, "train_loss": 0.00028808380011469126, "cos_r_negg": [0.5631439089775085, 0.6350460052490234, 0.6772650480270386], "cos_innovation_negg": [0.5631439089775085, 0.6350460052490234, 0.6772650480270386], "cos_apical_negg": [0.5631439089775085, 0.6350460052490234, 0.6772650480270386], "cos_Ac_negg": [0.5631439089775085, 0.6350460052490234, 0.6772650480270386], "r_norm": [0.1378626674413681, 0.1347978413105011, 0.13628838956356049]}, {"step": 6900, "epoch": 17, "train_loss": 0.0006753129418939352, "cos_r_negg": [0.5636586546897888, 0.6349329948425293, 0.6774997711181641], "cos_innovation_negg": [0.5636586546897888, 0.6349329948425293, 0.6774997711181641], "cos_apical_negg": [0.5636586546897888, 0.6349329948425293, 0.6774997711181641], "cos_Ac_negg": [0.5636586546897888, 0.6349329948425293, 0.6774997711181641], "r_norm": [0.13729354739189148, 0.13425689935684204, 0.135767862200737]}, {"step": 6950, "epoch": 17, "train_loss": 0.0016183361876755953, "cos_r_negg": [0.5632232427597046, 0.6352022886276245, 0.677072286605835], "cos_innovation_negg": [0.5632232427597046, 0.6352022886276245, 0.677072286605835], "cos_apical_negg": [0.5632232427597046, 0.6352022886276245, 0.677072286605835], "cos_Ac_negg": [0.5632232427597046, 0.6352022886276245, 0.677072286605835], "r_norm": [0.13554298877716064, 0.13263463973999023, 0.13418278098106384]}, {"step": 7000, "epoch": 17, "train_loss": 0.0003361683920957148, "cos_r_negg": [0.5636635422706604, 0.6349698305130005, 0.6774451732635498], "cos_innovation_negg": [0.5636635422706604, 0.6349698305130005, 0.6774451732635498], "cos_apical_negg": [0.5636635422706604, 0.6349698305130005, 0.6774451732635498], "cos_Ac_negg": [0.5636635422706604, 0.6349698305130005, 0.6774451732635498], "r_norm": [0.13537126779556274, 0.13247688114643097, 0.13400086760520935]}, {"epoch_end": 17, "step": 7038, "test_acc": 0.971, "test_loss": 0.16477055702209473}, {"step": 7050, "epoch": 18, "train_loss": 0.00012481570593081415, "cos_r_negg": [0.5634878873825073, 0.6346147060394287, 0.677024245262146], "cos_innovation_negg": [0.5634878873825073, 0.6346147060394287, 0.677024245262146], "cos_apical_negg": [0.5634878873825073, 0.6346147060394287, 0.677024245262146], "cos_Ac_negg": [0.5634878873825073, 0.6346147060394287, 0.677024245262146], "r_norm": [0.1373751163482666, 0.1343066692352295, 0.1359834372997284]}, {"step": 7100, "epoch": 18, "train_loss": 0.00041181338019669056, "cos_r_negg": [0.5638604164123535, 0.6344518661499023, 0.6775900721549988], "cos_innovation_negg": [0.5638604164123535, 0.6344518661499023, 0.6775900721549988], "cos_apical_negg": [0.5638604164123535, 0.6344518661499023, 0.6775900721549988], "cos_Ac_negg": [0.5638604164123535, 0.6344518661499023, 0.6775900721549988], "r_norm": [0.1371658891439438, 0.13413143157958984, 0.13581494987010956]}, {"step": 7150, "epoch": 18, "train_loss": 0.0002750277635641396, "cos_r_negg": [0.56539386510849, 0.635069727897644, 0.67767733335495], "cos_innovation_negg": [0.56539386510849, 0.635069727897644, 0.67767733335495], "cos_apical_negg": [0.56539386510849, 0.635069727897644, 0.67767733335495], "cos_Ac_negg": [0.56539386510849, 0.635069727897644, 0.67767733335495], "r_norm": [0.13690973818302155, 0.1338893473148346, 0.1355157345533371]}, {"step": 7200, "epoch": 18, "train_loss": 0.00028479346656240523, "cos_r_negg": [0.5658869743347168, 0.6348302364349365, 0.677764356136322], "cos_innovation_negg": [0.5658869743347168, 0.6348302364349365, 0.677764356136322], "cos_apical_negg": [0.5658869743347168, 0.6348302364349365, 0.677764356136322], "cos_Ac_negg": [0.5658869743347168, 0.6348302364349365, 0.677764356136322], "r_norm": [0.13684499263763428, 0.1338140368461609, 0.13539598882198334]}, {"step": 7250, "epoch": 18, "train_loss": 0.0003868063213303685, "cos_r_negg": [0.5657260417938232, 0.6343393921852112, 0.677925705909729], "cos_innovation_negg": [0.5657260417938232, 0.6343393921852112, 0.677925705909729], "cos_apical_negg": [0.5657260417938232, 0.6343393921852112, 0.677925705909729], "cos_Ac_negg": [0.5657260417938232, 0.6343393921852112, 0.677925705909729], "r_norm": [0.1365097016096115, 0.13354089856147766, 0.13505777716636658]}, {"step": 7300, "epoch": 18, "train_loss": 0.00011705293582053855, "cos_r_negg": [0.5656157732009888, 0.6348361968994141, 0.6773326396942139], "cos_innovation_negg": [0.5656157732009888, 0.6348361968994141, 0.6773326396942139], "cos_apical_negg": [0.5656157732009888, 0.6348361968994141, 0.6773326396942139], "cos_Ac_negg": [0.5656157732009888, 0.6348361968994141, 0.6773326396942139], "r_norm": [0.13574737310409546, 0.13277365267276764, 0.13435842096805573]}, {"step": 7350, "epoch": 18, "train_loss": 0.0007820988539606333, "cos_r_negg": [0.5650259256362915, 0.6353505253791809, 0.6782957315444946], "cos_innovation_negg": [0.5650259256362915, 0.6353505253791809, 0.6782957315444946], "cos_apical_negg": [0.5650259256362915, 0.6353505253791809, 0.6782957315444946], "cos_Ac_negg": [0.5650259256362915, 0.6353505253791809, 0.6782957315444946], "r_norm": [0.13679230213165283, 0.1336785852909088, 0.1353691667318344]}, {"step": 7400, "epoch": 18, "train_loss": 0.00016073111328296363, "cos_r_negg": [0.5649049282073975, 0.635573148727417, 0.6778131723403931], "cos_innovation_negg": [0.5649049282073975, 0.635573148727417, 0.6778131723403931], "cos_apical_negg": [0.5649049282073975, 0.635573148727417, 0.6778131723403931], "cos_Ac_negg": [0.5649049282073975, 0.635573148727417, 0.6778131723403931], "r_norm": [0.13672402501106262, 0.1336272656917572, 0.13530410826206207]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9708, "test_loss": 0.16637429275512694}, {"step": 7450, "epoch": 19, "train_loss": 3.929401282221079e-05, "cos_r_negg": [0.5641429424285889, 0.6345242857933044, 0.6771981716156006], "cos_innovation_negg": [0.5641429424285889, 0.6345242857933044, 0.6771981716156006], "cos_apical_negg": [0.5641429424285889, 0.6345242857933044, 0.6771981716156006], "cos_Ac_negg": [0.5641429424285889, 0.6345242857933044, 0.6771981716156006], "r_norm": [0.13515602052211761, 0.13221533596515656, 0.13383296132087708]}, {"step": 7500, "epoch": 19, "train_loss": 0.000350951129803434, "cos_r_negg": [0.5643362998962402, 0.6347794532775879, 0.6778059601783752], "cos_innovation_negg": [0.5643362998962402, 0.6347794532775879, 0.6778059601783752], "cos_apical_negg": [0.5643362998962402, 0.6347794532775879, 0.6778059601783752], "cos_Ac_negg": [0.5643362998962402, 0.6347794532775879, 0.6778059601783752], "r_norm": [0.1361807882785797, 0.13323047757148743, 0.1348324716091156]}, {"step": 7550, "epoch": 19, "train_loss": 0.00015385176811832935, "cos_r_negg": [0.5652161836624146, 0.6354360580444336, 0.677256166934967], "cos_innovation_negg": [0.5652161836624146, 0.6354360580444336, 0.677256166934967], "cos_apical_negg": [0.5652161836624146, 0.6354360580444336, 0.677256166934967], "cos_Ac_negg": [0.5652161836624146, 0.6354360580444336, 0.677256166934967], "r_norm": [0.13577690720558167, 0.1328345537185669, 0.13440868258476257]}, {"step": 7600, "epoch": 19, "train_loss": 0.00025666216970421374, "cos_r_negg": [0.5649275779724121, 0.6354023814201355, 0.6775671243667603], "cos_innovation_negg": [0.5649275779724121, 0.6354023814201355, 0.6775671243667603], "cos_apical_negg": [0.5649275779724121, 0.6354023814201355, 0.6775671243667603], "cos_Ac_negg": [0.5649275779724121, 0.6354023814201355, 0.6775671243667603], "r_norm": [0.13692399859428406, 0.13386160135269165, 0.13547813892364502]}, {"step": 7650, "epoch": 19, "train_loss": 0.0002460424730088562, "cos_r_negg": [0.5654614567756653, 0.6351456642150879, 0.6778969764709473], "cos_innovation_negg": [0.5654614567756653, 0.6351456642150879, 0.6778969764709473], "cos_apical_negg": [0.5654614567756653, 0.6351456642150879, 0.6778969764709473], "cos_Ac_negg": [0.5654614567756653, 0.6351456642150879, 0.6778969764709473], "r_norm": [0.13755711913108826, 0.13441939651966095, 0.13613075017929077]}, {"step": 7700, "epoch": 19, "train_loss": 0.0003748289600480348, "cos_r_negg": [0.5662862062454224, 0.6354097127914429, 0.6782540082931519], "cos_innovation_negg": [0.5662862062454224, 0.6354097127914429, 0.6782540082931519], "cos_apical_negg": [0.5662862062454224, 0.6354097127914429, 0.6782540082931519], "cos_Ac_negg": [0.5662862062454224, 0.6354097127914429, 0.6782540082931519], "r_norm": [0.13674023747444153, 0.13366694748401642, 0.13535799086093903]}, {"step": 7750, "epoch": 19, "train_loss": 0.0006176414899528027, "cos_r_negg": [0.5658199191093445, 0.6362113952636719, 0.6788793206214905], "cos_innovation_negg": [0.5658199191093445, 0.6362113952636719, 0.6788793206214905], "cos_apical_negg": [0.5658199191093445, 0.6362113952636719, 0.6788793206214905], "cos_Ac_negg": [0.5658199191093445, 0.6362113952636719, 0.6788793206214905], "r_norm": [0.13637889921665192, 0.1333383321762085, 0.13499385118484497]}, {"step": 7800, "epoch": 19, "train_loss": 0.00025995535543188453, "cos_r_negg": [0.5665167570114136, 0.6363417506217957, 0.6792000532150269], "cos_innovation_negg": [0.5665167570114136, 0.6363417506217957, 0.6792000532150269], "cos_apical_negg": [0.5665167570114136, 0.6363417506217957, 0.6792000532150269], "cos_Ac_negg": [0.5665167570114136, 0.6363417506217957, 0.6792000532150269], "r_norm": [0.13732896745204926, 0.13425345718860626, 0.1359100490808487]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.971, "test_loss": 0.1685058780670166}, {"step": 7850, "epoch": 20, "train_loss": 0.0003030593506991863, "cos_r_negg": [0.5657829642295837, 0.6355138421058655, 0.679034948348999], "cos_innovation_negg": [0.5657829642295837, 0.6355138421058655, 0.679034948348999], "cos_apical_negg": [0.5657829642295837, 0.6355138421058655, 0.679034948348999], "cos_Ac_negg": [0.5657829642295837, 0.6355138421058655, 0.679034948348999], "r_norm": [0.13718411326408386, 0.13408327102661133, 0.13577678799629211]}, {"step": 7900, "epoch": 20, "train_loss": 0.00029655167600139976, "cos_r_negg": [0.5645723342895508, 0.6346889734268188, 0.6783400774002075], "cos_innovation_negg": [0.5645723342895508, 0.6346889734268188, 0.6783400774002075], "cos_apical_negg": [0.5645723342895508, 0.6346889734268188, 0.6783400774002075], "cos_Ac_negg": [0.5645723342895508, 0.6346889734268188, 0.6783400774002075], "r_norm": [0.13647347688674927, 0.13340824842453003, 0.1351117640733719]}, {"step": 7950, "epoch": 20, "train_loss": 0.00030222232453525066, "cos_r_negg": [0.5641942620277405, 0.6348505020141602, 0.6781685948371887], "cos_innovation_negg": [0.5641942620277405, 0.6348505020141602, 0.6781685948371887], "cos_apical_negg": [0.5641942620277405, 0.6348505020141602, 0.6781685948371887], "cos_Ac_negg": [0.5641942620277405, 0.6348505020141602, 0.6781685948371887], "r_norm": [0.13626420497894287, 0.13324853777885437, 0.13490676879882812]}, {"step": 8000, "epoch": 20, "train_loss": 0.0001915910397656262, "cos_r_negg": [0.563732922077179, 0.6348075866699219, 0.6781225204467773], "cos_innovation_negg": [0.563732922077179, 0.6348075866699219, 0.6781225204467773], "cos_apical_negg": [0.563732922077179, 0.6348075866699219, 0.6781225204467773], "cos_Ac_negg": [0.563732922077179, 0.6348075866699219, 0.6781225204467773], "r_norm": [0.13692787289619446, 0.13385501503944397, 0.13552626967430115]}, {"step": 8050, "epoch": 20, "train_loss": 0.0009336535586044192, "cos_r_negg": [0.5638518333435059, 0.6352275609970093, 0.6784844398498535], "cos_innovation_negg": [0.5638518333435059, 0.6352275609970093, 0.6784844398498535], "cos_apical_negg": [0.5638518333435059, 0.6352275609970093, 0.6784844398498535], "cos_Ac_negg": [0.5638518333435059, 0.6352275609970093, 0.6784844398498535], "r_norm": [0.13737134635448456, 0.1342926323413849, 0.13588935136795044]}, {"step": 8100, "epoch": 20, "train_loss": 0.00029304728377610445, "cos_r_negg": [0.5646332502365112, 0.6355032920837402, 0.6788865327835083], "cos_innovation_negg": [0.5646332502365112, 0.6355032920837402, 0.6788865327835083], "cos_apical_negg": [0.5646332502365112, 0.6355032920837402, 0.6788865327835083], "cos_Ac_negg": [0.5646332502365112, 0.6355032920837402, 0.6788865327835083], "r_norm": [0.13778066635131836, 0.13464637100696564, 0.13627710938453674]}, {"step": 8150, "epoch": 20, "train_loss": 0.00032788398675620556, "cos_r_negg": [0.5633751153945923, 0.634724497795105, 0.6782124638557434], "cos_innovation_negg": [0.5633751153945923, 0.634724497795105, 0.6782124638557434], "cos_apical_negg": [0.5633751153945923, 0.634724497795105, 0.6782124638557434], "cos_Ac_negg": [0.5633751153945923, 0.634724497795105, 0.6782124638557434], "r_norm": [0.13705426454544067, 0.13391584157943726, 0.1356140673160553]}, {"step": 8200, "epoch": 20, "train_loss": 0.00027853468782268465, "cos_r_negg": [0.563408613204956, 0.6347285509109497, 0.6776638031005859], "cos_innovation_negg": [0.563408613204956, 0.6347285509109497, 0.6776638031005859], "cos_apical_negg": [0.563408613204956, 0.6347285509109497, 0.6776638031005859], "cos_Ac_negg": [0.563408613204956, 0.6347285509109497, 0.6776638031005859], "r_norm": [0.1366090178489685, 0.1335153728723526, 0.13522806763648987]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9707, "test_loss": 0.16925469856262207}, {"step": 8250, "epoch": 21, "train_loss": 0.00039176925201900303, "cos_r_negg": [0.5636810660362244, 0.6347805261611938, 0.6778781414031982], "cos_innovation_negg": [0.5636810660362244, 0.6347805261611938, 0.6778781414031982], "cos_apical_negg": [0.5636810660362244, 0.6347805261611938, 0.6778781414031982], "cos_Ac_negg": [0.5636810660362244, 0.6347805261611938, 0.6778781414031982], "r_norm": [0.13728588819503784, 0.13415475189685822, 0.13591547310352325]}, {"step": 8300, "epoch": 21, "train_loss": 0.0004046795074827969, "cos_r_negg": [0.5633439421653748, 0.6347760558128357, 0.6778717041015625], "cos_innovation_negg": [0.5633439421653748, 0.6347760558128357, 0.6778717041015625], "cos_apical_negg": [0.5633439421653748, 0.6347760558128357, 0.6778717041015625], "cos_Ac_negg": [0.5633439421653748, 0.6347760558128357, 0.6778717041015625], "r_norm": [0.13623926043510437, 0.1331729143857956, 0.1348845660686493]}, {"step": 8350, "epoch": 21, "train_loss": 0.000358615507138893, "cos_r_negg": [0.5634841322898865, 0.6354883909225464, 0.6780835390090942], "cos_innovation_negg": [0.5634841322898865, 0.6354883909225464, 0.6780835390090942], "cos_apical_negg": [0.5634841322898865, 0.6354883909225464, 0.6780835390090942], "cos_Ac_negg": [0.5634841322898865, 0.6354883909225464, 0.6780835390090942], "r_norm": [0.13665910065174103, 0.1335904598236084, 0.1352577805519104]}, {"step": 8400, "epoch": 21, "train_loss": 0.000410301610827446, "cos_r_negg": [0.5642249584197998, 0.6354526281356812, 0.6784869432449341], "cos_innovation_negg": [0.5642249584197998, 0.6354526281356812, 0.6784869432449341], "cos_apical_negg": [0.5642249584197998, 0.6354526281356812, 0.6784869432449341], "cos_Ac_negg": [0.5642249584197998, 0.6354526281356812, 0.6784869432449341], "r_norm": [0.13594716787338257, 0.13295529782772064, 0.13452963531017303]}, {"step": 8450, "epoch": 21, "train_loss": 0.00022546654508914798, "cos_r_negg": [0.5639657974243164, 0.6349834203720093, 0.678264856338501], "cos_innovation_negg": [0.5639657974243164, 0.6349834203720093, 0.678264856338501], "cos_apical_negg": [0.5639657974243164, 0.6349834203720093, 0.678264856338501], "cos_Ac_negg": [0.5639657974243164, 0.6349834203720093, 0.678264856338501], "r_norm": [0.13448983430862427, 0.13157570362091064, 0.13312551379203796]}, {"step": 8500, "epoch": 21, "train_loss": 0.0002512058708816767, "cos_r_negg": [0.5636979937553406, 0.6353070735931396, 0.6781126856803894], "cos_innovation_negg": [0.5636979937553406, 0.6353070735931396, 0.6781126856803894], "cos_apical_negg": [0.5636979937553406, 0.6353070735931396, 0.6781126856803894], "cos_Ac_negg": [0.5636979937553406, 0.6353070735931396, 0.6781126856803894], "r_norm": [0.1363544911146164, 0.13333158195018768, 0.13491609692573547]}, {"step": 8550, "epoch": 21, "train_loss": 0.00019407675426919013, "cos_r_negg": [0.5641385316848755, 0.6354917287826538, 0.6783136129379272], "cos_innovation_negg": [0.5641385316848755, 0.6354917287826538, 0.6783136129379272], "cos_apical_negg": [0.5641385316848755, 0.6354917287826538, 0.6783136129379272], "cos_Ac_negg": [0.5641385316848755, 0.6354917287826538, 0.6783136129379272], "r_norm": [0.13840316236019135, 0.13525313138961792, 0.1369209736585617]}, {"step": 8600, "epoch": 21, "train_loss": 0.0006775517831556499, "cos_r_negg": [0.5630521774291992, 0.634575366973877, 0.6781271696090698], "cos_innovation_negg": [0.5630521774291992, 0.634575366973877, 0.6781271696090698], "cos_apical_negg": [0.5630521774291992, 0.634575366973877, 0.6781271696090698], "cos_Ac_negg": [0.5630521774291992, 0.634575366973877, 0.6781271696090698], "r_norm": [0.13711541891098022, 0.13396064937114716, 0.1357196867465973]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9709, "test_loss": 0.1706870838165283}, {"step": 8650, "epoch": 22, "train_loss": 0.0001784804044291377, "cos_r_negg": [0.5620522499084473, 0.634259819984436, 0.6781629920005798], "cos_innovation_negg": [0.5620522499084473, 0.634259819984436, 0.6781629920005798], "cos_apical_negg": [0.5620522499084473, 0.634259819984436, 0.6781629920005798], "cos_Ac_negg": [0.5620522499084473, 0.634259819984436, 0.6781629920005798], "r_norm": [0.13653221726417542, 0.1333995908498764, 0.13516990840435028]}, {"step": 8700, "epoch": 22, "train_loss": 0.00014866446144878864, "cos_r_negg": [0.5623602867126465, 0.6349677443504333, 0.6777766346931458], "cos_innovation_negg": [0.5623602867126465, 0.6349677443504333, 0.6777766346931458], "cos_apical_negg": [0.5623602867126465, 0.6349677443504333, 0.6777766346931458], "cos_Ac_negg": [0.5623602867126465, 0.6349677443504333, 0.6777766346931458], "r_norm": [0.1366533637046814, 0.13351619243621826, 0.13527940213680267]}, {"step": 8750, "epoch": 22, "train_loss": 0.00033025554148480296, "cos_r_negg": [0.5622419118881226, 0.6346672773361206, 0.6784358024597168], "cos_innovation_negg": [0.5622419118881226, 0.6346672773361206, 0.6784358024597168], "cos_apical_negg": [0.5622419118881226, 0.6346672773361206, 0.6784358024597168], "cos_Ac_negg": [0.5622419118881226, 0.6346672773361206, 0.6784358024597168], "r_norm": [0.1366216242313385, 0.133529931306839, 0.13527627289295197]}, {"step": 8800, "epoch": 22, "train_loss": 0.00043997541069984436, "cos_r_negg": [0.5623010993003845, 0.6346597671508789, 0.6785541772842407], "cos_innovation_negg": [0.5623010993003845, 0.6346597671508789, 0.6785541772842407], "cos_apical_negg": [0.5623010993003845, 0.6346597671508789, 0.6785541772842407], "cos_Ac_negg": [0.5623010993003845, 0.6346597671508789, 0.6785541772842407], "r_norm": [0.136732280254364, 0.13367405533790588, 0.13534288108348846]}, {"step": 8850, "epoch": 22, "train_loss": 0.00019261802663095295, "cos_r_negg": [0.5626536011695862, 0.6340487599372864, 0.6781031489372253], "cos_innovation_negg": [0.5626536011695862, 0.6340487599372864, 0.6781031489372253], "cos_apical_negg": [0.5626536011695862, 0.6340487599372864, 0.6781031489372253], "cos_Ac_negg": [0.5626536011695862, 0.6340487599372864, 0.6781031489372253], "r_norm": [0.13576078414916992, 0.13271702826023102, 0.13441047072410583]}, {"step": 8900, "epoch": 22, "train_loss": 0.0004908476257696748, "cos_r_negg": [0.5616327524185181, 0.6338987350463867, 0.6776952743530273], "cos_innovation_negg": [0.5616327524185181, 0.6338987350463867, 0.6776952743530273], "cos_apical_negg": [0.5616327524185181, 0.6338987350463867, 0.6776952743530273], "cos_Ac_negg": [0.5616327524185181, 0.6338987350463867, 0.6776952743530273], "r_norm": [0.1360866278409958, 0.1330563724040985, 0.1347229778766632]}, {"step": 8950, "epoch": 22, "train_loss": 0.00021805783035233617, "cos_r_negg": [0.5617178678512573, 0.6342891454696655, 0.6776261329650879], "cos_innovation_negg": [0.5617178678512573, 0.6342891454696655, 0.6776261329650879], "cos_apical_negg": [0.5617178678512573, 0.6342891454696655, 0.6776261329650879], "cos_Ac_negg": [0.5617178678512573, 0.6342891454696655, 0.6776261329650879], "r_norm": [0.1367773413658142, 0.13371062278747559, 0.13541805744171143]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9708, "test_loss": 0.17192392330169678}, {"step": 9000, "epoch": 23, "train_loss": 0.00011313322465866804, "cos_r_negg": [0.5611909627914429, 0.6349185705184937, 0.6776412129402161], "cos_innovation_negg": [0.5611909627914429, 0.6349185705184937, 0.6776412129402161], "cos_apical_negg": [0.5611909627914429, 0.6349185705184937, 0.6776412129402161], "cos_Ac_negg": [0.5611909627914429, 0.6349185705184937, 0.6776412129402161], "r_norm": [0.13623926043510437, 0.1331460177898407, 0.13489648699760437]}, {"step": 9050, "epoch": 23, "train_loss": 0.0002899000246543437, "cos_r_negg": [0.5613402724266052, 0.6351237893104553, 0.6777890920639038], "cos_innovation_negg": [0.5613402724266052, 0.6351237893104553, 0.6777890920639038], "cos_apical_negg": [0.5613402724266052, 0.6351237893104553, 0.6777890920639038], "cos_Ac_negg": [0.5613402724266052, 0.6351237893104553, 0.6777890920639038], "r_norm": [0.13728131353855133, 0.13410493731498718, 0.1358623206615448]}, {"step": 9100, "epoch": 23, "train_loss": 0.00010537832713453099, "cos_r_negg": [0.5619636178016663, 0.6354840397834778, 0.6779921054840088], "cos_innovation_negg": [0.5619636178016663, 0.6354840397834778, 0.6779921054840088], "cos_apical_negg": [0.5619636178016663, 0.6354840397834778, 0.6779921054840088], "cos_Ac_negg": [0.5619636178016663, 0.6354840397834778, 0.6779921054840088], "r_norm": [0.13640688359737396, 0.13329017162322998, 0.1349908709526062]}, {"step": 9150, "epoch": 23, "train_loss": 0.00038587464950978756, "cos_r_negg": [0.5616177320480347, 0.635329008102417, 0.6779994964599609], "cos_innovation_negg": [0.5616177320480347, 0.635329008102417, 0.6779994964599609], "cos_apical_negg": [0.5616177320480347, 0.635329008102417, 0.6779994964599609], "cos_Ac_negg": [0.5616177320480347, 0.635329008102417, 0.6779994964599609], "r_norm": [0.13641509413719177, 0.1333278864622116, 0.13503335416316986]}, {"step": 9200, "epoch": 23, "train_loss": 0.00027231863350607455, "cos_r_negg": [0.5620118975639343, 0.6354725360870361, 0.6783305406570435], "cos_innovation_negg": [0.5620118975639343, 0.6354725360870361, 0.6783305406570435], "cos_apical_negg": [0.5620118975639343, 0.6354725360870361, 0.6783305406570435], "cos_Ac_negg": [0.5620118975639343, 0.6354725360870361, 0.6783305406570435], "r_norm": [0.13638465106487274, 0.1332935392856598, 0.13498157262802124]}, {"step": 9250, "epoch": 23, "train_loss": 0.0002090476918965578, "cos_r_negg": [0.5608701705932617, 0.634968101978302, 0.6785139441490173], "cos_innovation_negg": [0.5608701705932617, 0.634968101978302, 0.6785139441490173], "cos_apical_negg": [0.5608701705932617, 0.634968101978302, 0.6785139441490173], "cos_Ac_negg": [0.5608701705932617, 0.634968101978302, 0.6785139441490173], "r_norm": [0.13564664125442505, 0.13263891637325287, 0.13427351415157318]}, {"step": 9300, "epoch": 23, "train_loss": 0.0001533524045953527, "cos_r_negg": [0.5610320568084717, 0.6341733932495117, 0.6776304841041565], "cos_innovation_negg": [0.5610320568084717, 0.6341733932495117, 0.6776304841041565], "cos_apical_negg": [0.5610320568084717, 0.6341733932495117, 0.6776304841041565], "cos_Ac_negg": [0.5610320568084717, 0.6341733932495117, 0.6776304841041565], "r_norm": [0.13548272848129272, 0.13244286179542542, 0.13416415452957153]}, {"step": 9350, "epoch": 23, "train_loss": 0.0002520310808904469, "cos_r_negg": [0.5602407455444336, 0.6338412761688232, 0.6779542565345764], "cos_innovation_negg": [0.5602407455444336, 0.6338412761688232, 0.6779542565345764], "cos_apical_negg": [0.5602407455444336, 0.6338412761688232, 0.6779542565345764], "cos_Ac_negg": [0.5602407455444336, 0.6338412761688232, 0.6779542565345764], "r_norm": [0.13522067666053772, 0.13222144544124603, 0.13393498957157135]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9708, "test_loss": 0.17338037719726562}, {"step": 9400, "epoch": 24, "train_loss": 0.00020853467867709696, "cos_r_negg": [0.5604872107505798, 0.6342915892601013, 0.6771618723869324], "cos_innovation_negg": [0.5604872107505798, 0.6342915892601013, 0.6771618723869324], "cos_apical_negg": [0.5604872107505798, 0.6342915892601013, 0.6771618723869324], "cos_Ac_negg": [0.5604872107505798, 0.6342915892601013, 0.6771618723869324], "r_norm": [0.13589143753051758, 0.13283929228782654, 0.13459154963493347]}, {"step": 9450, "epoch": 24, "train_loss": 8.502665150444955e-05, "cos_r_negg": [0.5616112947463989, 0.6346249580383301, 0.6777513027191162], "cos_innovation_negg": [0.5616112947463989, 0.6346249580383301, 0.6777513027191162], "cos_apical_negg": [0.5616112947463989, 0.6346249580383301, 0.6777513027191162], "cos_Ac_negg": [0.5616112947463989, 0.6346249580383301, 0.6777513027191162], "r_norm": [0.13639290630817413, 0.13332922756671906, 0.13509687781333923]}, {"step": 9500, "epoch": 24, "train_loss": 0.00017350744747091085, "cos_r_negg": [0.562523365020752, 0.6349424123764038, 0.6783977746963501], "cos_innovation_negg": [0.562523365020752, 0.6349424123764038, 0.6783977746963501], "cos_apical_negg": [0.562523365020752, 0.6349424123764038, 0.6783977746963501], "cos_Ac_negg": [0.562523365020752, 0.6349424123764038, 0.6783977746963501], "r_norm": [0.13533207774162292, 0.13228920102119446, 0.13405272364616394]}, {"step": 9550, "epoch": 24, "train_loss": 0.0001283001620322466, "cos_r_negg": [0.5621028542518616, 0.6349138021469116, 0.6780773401260376], "cos_innovation_negg": [0.5621028542518616, 0.6349138021469116, 0.6780773401260376], "cos_apical_negg": [0.5621028542518616, 0.6349138021469116, 0.6780773401260376], "cos_Ac_negg": [0.5621028542518616, 0.6349138021469116, 0.6780773401260376], "r_norm": [0.13608640432357788, 0.13295696675777435, 0.1347428858280182]}, {"step": 9600, "epoch": 24, "train_loss": 0.00013139753718860447, "cos_r_negg": [0.5601425170898438, 0.6343356966972351, 0.6785479784011841], "cos_innovation_negg": [0.5601425170898438, 0.6343356966972351, 0.6785479784011841], "cos_apical_negg": [0.5601425170898438, 0.6343356966972351, 0.6785479784011841], "cos_Ac_negg": [0.5601425170898438, 0.6343356966972351, 0.6785479784011841], "r_norm": [0.1360991895198822, 0.13301381468772888, 0.134749636054039]}, {"step": 9650, "epoch": 24, "train_loss": 0.00028881418984383345, "cos_r_negg": [0.562812089920044, 0.634817361831665, 0.6790305376052856], "cos_innovation_negg": [0.562812089920044, 0.634817361831665, 0.6790305376052856], "cos_apical_negg": [0.562812089920044, 0.634817361831665, 0.6790305376052856], "cos_Ac_negg": [0.562812089920044, 0.634817361831665, 0.6790305376052856], "r_norm": [0.135869562625885, 0.13280583918094635, 0.13454368710517883]}, {"step": 9700, "epoch": 24, "train_loss": 0.000188079837244004, "cos_r_negg": [0.5623242855072021, 0.6348735094070435, 0.6786503195762634], "cos_innovation_negg": [0.5623242855072021, 0.6348735094070435, 0.6786503195762634], "cos_apical_negg": [0.5623242855072021, 0.6348735094070435, 0.6786503195762634], "cos_Ac_negg": [0.5623242855072021, 0.6348735094070435, 0.6786503195762634], "r_norm": [0.13636457920074463, 0.13326561450958252, 0.13500651717185974]}, {"step": 9750, "epoch": 24, "train_loss": 0.00033166760113090277, "cos_r_negg": [0.5617154836654663, 0.6348583698272705, 0.6791277527809143], "cos_innovation_negg": [0.5617154836654663, 0.6348583698272705, 0.6791277527809143], "cos_apical_negg": [0.5617154836654663, 0.6348583698272705, 0.6791277527809143], "cos_Ac_negg": [0.5617154836654663, 0.6348583698272705, 0.6791277527809143], "r_norm": [0.13671758770942688, 0.13362640142440796, 0.13536711037158966]}, {"epoch_end": 24, "step": 9775, "test_acc": 0.971, "test_loss": 0.17411411781311034}], "final": {"test_acc": 0.971, "test_loss": 0.17411411781311034, "wall_s": 17.66196322441101, "cos_r_negg": [0.5609441995620728, 0.6349866390228271, 0.6787049174308777], "cos_innovation_negg": [0.5609441995620728, 0.6349866390228271, 0.6787049174308777], "cos_apical_negg": [0.5609441995620728, 0.6349866390228271, 0.6787049174308777], "cos_Ac_negg": [0.5609441995620728, 0.6349866390228271, 0.6787049174308777]}} \ No newline at end of file diff --git a/results/ep_match_v1_mnist_dfa_w256_d3_s1.json b/results/ep_match_v1_mnist_dfa_w256_d3_s1.json new file mode 100644 index 0000000..b0beceb --- /dev/null +++ b/results/ep_match_v1_mnist_dfa_w256_d3_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_match_v1_mnist_dfa_w256_d3_s1", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3325631618499756, "cos_r_negg": [0.01792173460125923, -0.011461898684501648, -0.006219223607331514], "cos_innovation_negg": [0.01792173460125923, -0.011461898684501648, -0.006219223607331514], "cos_apical_negg": [0.01792173460125923, -0.011461898684501648, -0.006219223607331514], "cos_Ac_negg": [0.01792173460125923, -0.011461898684501648, -0.006219223607331514], "r_norm": [4.7435736656188965, 4.806507110595703, 4.796020984649658]}, {"step": 50, "epoch": 0, "train_loss": 0.7003065347671509, "cos_r_negg": [0.41564077138900757, 0.31092312932014465, 0.2914056181907654], "cos_innovation_negg": [0.41564077138900757, 0.31092312932014465, 0.2914056181907654], "cos_apical_negg": [0.41564077138900757, 0.31092312932014465, 0.2914056181907654], "cos_Ac_negg": [0.41564077138900757, 0.31092312932014465, 0.2914056181907654], "r_norm": [1.2285016775131226, 1.2330743074417114, 1.2591725587844849]}, {"step": 100, "epoch": 0, "train_loss": 0.3132488429546356, "cos_r_negg": [0.4717461168766022, 0.33066004514694214, 0.321431428194046], "cos_innovation_negg": [0.4717461168766022, 0.33066004514694214, 0.321431428194046], "cos_apical_negg": [0.4717461168766022, 0.33066004514694214, 0.321431428194046], "cos_Ac_negg": [0.4717461168766022, 0.33066004514694214, 0.321431428194046], "r_norm": [1.0010261535644531, 1.0069509744644165, 1.0258522033691406]}, {"step": 150, "epoch": 0, "train_loss": 0.17542332410812378, "cos_r_negg": [0.45539504289627075, 0.37492072582244873, 0.329465389251709], "cos_innovation_negg": [0.45539504289627075, 0.37492072582244873, 0.329465389251709], "cos_apical_negg": [0.45539504289627075, 0.37492072582244873, 0.329465389251709], "cos_Ac_negg": [0.45539504289627075, 0.37492072582244873, 0.329465389251709], "r_norm": [0.7834765911102295, 0.7896601557731628, 0.8006654381752014]}, {"step": 200, "epoch": 0, "train_loss": 0.2687198221683502, "cos_r_negg": [0.46881625056266785, 0.3943180739879608, 0.36964765191078186], "cos_innovation_negg": [0.46881625056266785, 0.3943180739879608, 0.36964765191078186], "cos_apical_negg": [0.46881625056266785, 0.3943180739879608, 0.36964765191078186], "cos_Ac_negg": [0.46881625056266785, 0.3943180739879608, 0.36964765191078186], "r_norm": [0.7462954521179199, 0.7534135580062866, 0.7658042907714844]}, {"step": 250, "epoch": 0, "train_loss": 0.30040496587753296, "cos_r_negg": [0.5090618133544922, 0.4221128821372986, 0.3860842287540436], "cos_innovation_negg": [0.5090618133544922, 0.4221128821372986, 0.3860842287540436], "cos_apical_negg": [0.5090618133544922, 0.4221128821372986, 0.3860842287540436], "cos_Ac_negg": [0.5090618133544922, 0.4221128821372986, 0.3860842287540436], "r_norm": [0.6461097002029419, 0.6488912105560303, 0.662902295589447]}, {"step": 300, "epoch": 0, "train_loss": 0.16167119145393372, "cos_r_negg": [0.49418580532073975, 0.4305404722690582, 0.38477909564971924], "cos_innovation_negg": [0.49418580532073975, 0.4305404722690582, 0.38477909564971924], "cos_apical_negg": [0.49418580532073975, 0.4305404722690582, 0.38477909564971924], "cos_Ac_negg": [0.49418580532073975, 0.4305404722690582, 0.38477909564971924], "r_norm": [0.593207597732544, 0.5925475358963013, 0.6079396605491638]}, {"step": 350, "epoch": 0, "train_loss": 0.19240573048591614, "cos_r_negg": [0.49054986238479614, 0.4248354434967041, 0.37794584035873413], "cos_innovation_negg": [0.49054986238479614, 0.4248354434967041, 0.37794584035873413], "cos_apical_negg": [0.49054986238479614, 0.4248354434967041, 0.37794584035873413], "cos_Ac_negg": [0.49054986238479614, 0.4248354434967041, 0.37794584035873413], "r_norm": [0.5731156468391418, 0.5790040493011475, 0.5856022834777832]}, {"epoch_end": 0, "step": 391, "test_acc": 0.9408, "test_loss": 0.19555468902587891}, {"step": 400, "epoch": 1, "train_loss": 0.14448067545890808, "cos_r_negg": [0.4839659035205841, 0.4413345158100128, 0.401180624961853], "cos_innovation_negg": [0.4839659035205841, 0.4413345158100128, 0.401180624961853], "cos_apical_negg": [0.4839659035205841, 0.4413345158100128, 0.401180624961853], "cos_Ac_negg": [0.4839659035205841, 0.4413345158100128, 0.401180624961853], "r_norm": [0.5443941354751587, 0.5488625764846802, 0.5549570918083191]}, {"step": 450, "epoch": 1, "train_loss": 0.1970212310552597, "cos_r_negg": [0.4902675151824951, 0.43865910172462463, 0.4123218357563019], "cos_innovation_negg": [0.4902675151824951, 0.43865910172462463, 0.4123218357563019], "cos_apical_negg": [0.4902675151824951, 0.43865910172462463, 0.4123218357563019], "cos_Ac_negg": [0.4902675151824951, 0.43865910172462463, 0.4123218357563019], "r_norm": [0.4583747386932373, 0.4588056802749634, 0.4669450521469116]}, {"step": 500, "epoch": 1, "train_loss": 0.1804504245519638, "cos_r_negg": [0.4853525161743164, 0.43966057896614075, 0.40529677271842957], "cos_innovation_negg": [0.4853525161743164, 0.43966057896614075, 0.40529677271842957], "cos_apical_negg": [0.4853525161743164, 0.43966057896614075, 0.40529677271842957], "cos_Ac_negg": [0.4853525161743164, 0.43966057896614075, 0.40529677271842957], "r_norm": [0.4043976664543152, 0.4060547947883606, 0.4114542007446289]}, {"step": 550, "epoch": 1, "train_loss": 0.16729575395584106, "cos_r_negg": [0.5106063485145569, 0.46283435821533203, 0.42025333642959595], "cos_innovation_negg": [0.5106063485145569, 0.46283435821533203, 0.42025333642959595], "cos_apical_negg": [0.5106063485145569, 0.46283435821533203, 0.42025333642959595], "cos_Ac_negg": [0.5106063485145569, 0.46283435821533203, 0.42025333642959595], "r_norm": [0.44027650356292725, 0.4402828514575958, 0.44952601194381714]}, {"step": 600, "epoch": 1, "train_loss": 0.10517038404941559, "cos_r_negg": [0.5110340118408203, 0.4846339821815491, 0.4244346618652344], "cos_innovation_negg": [0.5110340118408203, 0.4846339821815491, 0.4244346618652344], "cos_apical_negg": [0.5110340118408203, 0.4846339821815491, 0.4244346618652344], "cos_Ac_negg": [0.5110340118408203, 0.4846339821815491, 0.4244346618652344], "r_norm": [0.4804529547691345, 0.48491138219833374, 0.4852362275123596]}, {"step": 650, "epoch": 1, "train_loss": 0.13433139026165009, "cos_r_negg": [0.5102689266204834, 0.48502737283706665, 0.4243408441543579], "cos_innovation_negg": [0.5102689266204834, 0.48502737283706665, 0.4243408441543579], "cos_apical_negg": [0.5102689266204834, 0.48502737283706665, 0.4243408441543579], "cos_Ac_negg": [0.5102689266204834, 0.48502737283706665, 0.4243408441543579], "r_norm": [0.42958831787109375, 0.42960697412490845, 0.4351936876773834]}, {"step": 700, "epoch": 1, "train_loss": 0.14571471512317657, "cos_r_negg": [0.5016265511512756, 0.4781695306301117, 0.42138904333114624], "cos_innovation_negg": [0.5016265511512756, 0.4781695306301117, 0.42138904333114624], "cos_apical_negg": [0.5016265511512756, 0.4781695306301117, 0.42138904333114624], "cos_Ac_negg": [0.5016265511512756, 0.4781695306301117, 0.42138904333114624], "r_norm": [0.42952588200569153, 0.4340432286262512, 0.4343489706516266]}, {"step": 750, "epoch": 1, "train_loss": 0.1874101758003235, "cos_r_negg": [0.5318219065666199, 0.49775561690330505, 0.437970370054245], "cos_innovation_negg": [0.5318219065666199, 0.49775561690330505, 0.437970370054245], "cos_apical_negg": [0.5318219065666199, 0.49775561690330505, 0.437970370054245], "cos_Ac_negg": [0.5318219065666199, 0.49775561690330505, 0.437970370054245], "r_norm": [0.45617324113845825, 0.46354126930236816, 0.46587225794792175]}, {"epoch_end": 1, "step": 782, "test_acc": 0.9566, "test_loss": 0.14422514457702637}, {"step": 800, "epoch": 2, "train_loss": 0.10407431423664093, "cos_r_negg": [0.4896520972251892, 0.49045330286026, 0.44201433658599854], "cos_innovation_negg": [0.4896520972251892, 0.49045330286026, 0.44201433658599854], "cos_apical_negg": [0.4896520972251892, 0.49045330286026, 0.44201433658599854], "cos_Ac_negg": [0.4896520972251892, 0.49045330286026, 0.44201433658599854], "r_norm": [0.40256455540657043, 0.40530312061309814, 0.40898534655570984]}, {"step": 850, "epoch": 2, "train_loss": 0.07507171481847763, "cos_r_negg": [0.49808555841445923, 0.501785397529602, 0.451513409614563], "cos_innovation_negg": [0.49808555841445923, 0.501785397529602, 0.451513409614563], "cos_apical_negg": [0.49808555841445923, 0.501785397529602, 0.451513409614563], "cos_Ac_negg": [0.49808555841445923, 0.501785397529602, 0.451513409614563], "r_norm": [0.3472751975059509, 0.35244154930114746, 0.35382044315338135]}, {"step": 900, "epoch": 2, "train_loss": 0.0344250351190567, "cos_r_negg": [0.5227577686309814, 0.501377284526825, 0.4518999457359314], "cos_innovation_negg": [0.5227577686309814, 0.501377284526825, 0.4518999457359314], "cos_apical_negg": [0.5227577686309814, 0.501377284526825, 0.4518999457359314], "cos_Ac_negg": [0.5227577686309814, 0.501377284526825, 0.4518999457359314], "r_norm": [0.42026984691619873, 0.4199886620044708, 0.4236169755458832]}, {"step": 950, "epoch": 2, "train_loss": 0.13042908906936646, "cos_r_negg": [0.5394859910011292, 0.5285022258758545, 0.4527990221977234], "cos_innovation_negg": [0.5394859910011292, 0.5285022258758545, 0.4527990221977234], "cos_apical_negg": [0.5394859910011292, 0.5285022258758545, 0.4527990221977234], "cos_Ac_negg": [0.5394859910011292, 0.5285022258758545, 0.4527990221977234], "r_norm": [0.34777700901031494, 0.3503555655479431, 0.35071662068367004]}, {"step": 1000, "epoch": 2, "train_loss": 0.10429643094539642, "cos_r_negg": [0.547533392906189, 0.527867317199707, 0.4590854048728943], "cos_innovation_negg": [0.547533392906189, 0.527867317199707, 0.4590854048728943], "cos_apical_negg": [0.547533392906189, 0.527867317199707, 0.4590854048728943], "cos_Ac_negg": [0.547533392906189, 0.527867317199707, 0.4590854048728943], "r_norm": [0.4177277684211731, 0.4214029014110565, 0.4225059449672699]}, {"step": 1050, "epoch": 2, "train_loss": 0.10180195420980453, "cos_r_negg": [0.5220862627029419, 0.5125821828842163, 0.46975505352020264], "cos_innovation_negg": [0.5220862627029419, 0.5125821828842163, 0.46975505352020264], "cos_apical_negg": [0.5220862627029419, 0.5125821828842163, 0.46975505352020264], "cos_Ac_negg": [0.5220862627029419, 0.5125821828842163, 0.46975505352020264], "r_norm": [0.41370081901550293, 0.41286149621009827, 0.41980305314064026]}, {"step": 1100, "epoch": 2, "train_loss": 0.18512479960918427, "cos_r_negg": [0.5389041304588318, 0.5245411396026611, 0.46439048647880554], "cos_innovation_negg": [0.5389041304588318, 0.5245411396026611, 0.46439048647880554], "cos_apical_negg": [0.5389041304588318, 0.5245411396026611, 0.46439048647880554], "cos_Ac_negg": [0.5389041304588318, 0.5245411396026611, 0.46439048647880554], "r_norm": [0.3160831332206726, 0.3184950351715088, 0.3214266300201416]}, {"step": 1150, "epoch": 2, "train_loss": 0.13590753078460693, "cos_r_negg": [0.540425181388855, 0.5342485904693604, 0.47808021306991577], "cos_innovation_negg": [0.540425181388855, 0.5342485904693604, 0.47808021306991577], "cos_apical_negg": [0.540425181388855, 0.5342485904693604, 0.47808021306991577], "cos_Ac_negg": [0.540425181388855, 0.5342485904693604, 0.47808021306991577], "r_norm": [0.2944326102733612, 0.29455599188804626, 0.2989705204963684]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9619, "test_loss": 0.13187642669677735}, {"step": 1200, "epoch": 3, "train_loss": 0.12057094275951385, "cos_r_negg": [0.5511176586151123, 0.5367355346679688, 0.4918718934059143], "cos_innovation_negg": [0.5511176586151123, 0.5367355346679688, 0.4918718934059143], "cos_apical_negg": [0.5511176586151123, 0.5367355346679688, 0.4918718934059143], "cos_Ac_negg": [0.5511176586151123, 0.5367355346679688, 0.4918718934059143], "r_norm": [0.400901198387146, 0.40190786123275757, 0.40491366386413574]}, {"step": 1250, "epoch": 3, "train_loss": 0.09190027415752411, "cos_r_negg": [0.5394009947776794, 0.5413719415664673, 0.5010004043579102], "cos_innovation_negg": [0.5394009947776794, 0.5413719415664673, 0.5010004043579102], "cos_apical_negg": [0.5394009947776794, 0.5413719415664673, 0.5010004043579102], "cos_Ac_negg": [0.5394009947776794, 0.5413719415664673, 0.5010004043579102], "r_norm": [0.3479789197444916, 0.35174885392189026, 0.3544926047325134]}, {"step": 1300, "epoch": 3, "train_loss": 0.014962372370064259, "cos_r_negg": [0.532494306564331, 0.5313753485679626, 0.501737654209137], "cos_innovation_negg": [0.532494306564331, 0.5313753485679626, 0.501737654209137], "cos_apical_negg": [0.532494306564331, 0.5313753485679626, 0.501737654209137], "cos_Ac_negg": [0.532494306564331, 0.5313753485679626, 0.501737654209137], "r_norm": [0.3638189435005188, 0.368020236492157, 0.3692258596420288]}, {"step": 1350, "epoch": 3, "train_loss": 0.02610723488032818, "cos_r_negg": [0.5231947898864746, 0.5312000513076782, 0.5038590431213379], "cos_innovation_negg": [0.5231947898864746, 0.5312000513076782, 0.5038590431213379], "cos_apical_negg": [0.5231947898864746, 0.5312000513076782, 0.5038590431213379], "cos_Ac_negg": [0.5231947898864746, 0.5312000513076782, 0.5038590431213379], "r_norm": [0.3104226589202881, 0.30966877937316895, 0.3123634457588196]}, {"step": 1400, "epoch": 3, "train_loss": 0.03787291795015335, "cos_r_negg": [0.5413620471954346, 0.5382270812988281, 0.5062658786773682], "cos_innovation_negg": [0.5413620471954346, 0.5382270812988281, 0.5062658786773682], "cos_apical_negg": [0.5413620471954346, 0.5382270812988281, 0.5062658786773682], "cos_Ac_negg": [0.5413620471954346, 0.5382270812988281, 0.5062658786773682], "r_norm": [0.3370714485645294, 0.34009838104248047, 0.34341585636138916]}, {"step": 1450, "epoch": 3, "train_loss": 0.024879487231373787, "cos_r_negg": [0.5305036306381226, 0.5412448048591614, 0.5132496356964111], "cos_innovation_negg": [0.5305036306381226, 0.5412448048591614, 0.5132496356964111], "cos_apical_negg": [0.5305036306381226, 0.5412448048591614, 0.5132496356964111], "cos_Ac_negg": [0.5305036306381226, 0.5412448048591614, 0.5132496356964111], "r_norm": [0.3273622989654541, 0.3298325538635254, 0.33169177174568176]}, {"step": 1500, "epoch": 3, "train_loss": 0.07239923626184464, "cos_r_negg": [0.5462867617607117, 0.5573773384094238, 0.5096065998077393], "cos_innovation_negg": [0.5462867617607117, 0.5573773384094238, 0.5096065998077393], "cos_apical_negg": [0.5462867617607117, 0.5573773384094238, 0.5096065998077393], "cos_Ac_negg": [0.5462867617607117, 0.5573773384094238, 0.5096065998077393], "r_norm": [0.27496588230133057, 0.27469778060913086, 0.2757679522037506]}, {"step": 1550, "epoch": 3, "train_loss": 0.07345081865787506, "cos_r_negg": [0.5471221804618835, 0.5580427050590515, 0.5117762684822083], "cos_innovation_negg": [0.5471221804618835, 0.5580427050590515, 0.5117762684822083], "cos_apical_negg": [0.5471221804618835, 0.5580427050590515, 0.5117762684822083], "cos_Ac_negg": [0.5471221804618835, 0.5580427050590515, 0.5117762684822083], "r_norm": [0.2815324366092682, 0.280548632144928, 0.28271806240081787]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9639, "test_loss": 0.13285413398742676}, {"step": 1600, "epoch": 4, "train_loss": 0.04300045594573021, "cos_r_negg": [0.548197865486145, 0.5726513862609863, 0.5296549201011658], "cos_innovation_negg": [0.548197865486145, 0.5726513862609863, 0.5296549201011658], "cos_apical_negg": [0.548197865486145, 0.5726513862609863, 0.5296549201011658], "cos_Ac_negg": [0.548197865486145, 0.5726513862609863, 0.5296549201011658], "r_norm": [0.2795335054397583, 0.27886393666267395, 0.2816530466079712]}, {"step": 1650, "epoch": 4, "train_loss": 0.08055886626243591, "cos_r_negg": [0.5406944751739502, 0.5628681182861328, 0.5391868352890015], "cos_innovation_negg": [0.5406944751739502, 0.5628681182861328, 0.5391868352890015], "cos_apical_negg": [0.5406944751739502, 0.5628681182861328, 0.5391868352890015], "cos_Ac_negg": [0.5406944751739502, 0.5628681182861328, 0.5391868352890015], "r_norm": [0.2716537117958069, 0.27243471145629883, 0.2745879888534546]}, {"step": 1700, "epoch": 4, "train_loss": 0.05956104397773743, "cos_r_negg": [0.5515081882476807, 0.5637811422348022, 0.5425556302070618], "cos_innovation_negg": [0.5515081882476807, 0.5637811422348022, 0.5425556302070618], "cos_apical_negg": [0.5515081882476807, 0.5637811422348022, 0.5425556302070618], "cos_Ac_negg": [0.5515081882476807, 0.5637811422348022, 0.5425556302070618], "r_norm": [0.2538022994995117, 0.25588977336883545, 0.2582235336303711]}, {"step": 1750, "epoch": 4, "train_loss": 0.07891780138015747, "cos_r_negg": [0.5519150495529175, 0.5750154852867126, 0.5472017526626587], "cos_innovation_negg": [0.5519150495529175, 0.5750154852867126, 0.5472017526626587], "cos_apical_negg": [0.5519150495529175, 0.5750154852867126, 0.5472017526626587], "cos_Ac_negg": [0.5519150495529175, 0.5750154852867126, 0.5472017526626587], "r_norm": [0.23036059737205505, 0.2324325293302536, 0.2332795262336731]}, {"step": 1800, "epoch": 4, "train_loss": 0.030536454170942307, "cos_r_negg": [0.5560687780380249, 0.572013258934021, 0.5443101525306702], "cos_innovation_negg": [0.5560687780380249, 0.572013258934021, 0.5443101525306702], "cos_apical_negg": [0.5560687780380249, 0.572013258934021, 0.5443101525306702], "cos_Ac_negg": [0.5560687780380249, 0.572013258934021, 0.5443101525306702], "r_norm": [0.23143421113491058, 0.23177626729011536, 0.23493148386478424]}, {"step": 1850, "epoch": 4, "train_loss": 0.14382866024971008, "cos_r_negg": [0.5580585598945618, 0.5758178234100342, 0.546535313129425], "cos_innovation_negg": [0.5580585598945618, 0.5758178234100342, 0.546535313129425], "cos_apical_negg": [0.5580585598945618, 0.5758178234100342, 0.546535313129425], "cos_Ac_negg": [0.5580585598945618, 0.5758178234100342, 0.546535313129425], "r_norm": [0.2475300282239914, 0.24790319800376892, 0.25061315298080444]}, {"step": 1900, "epoch": 4, "train_loss": 0.018434131518006325, "cos_r_negg": [0.5609899759292603, 0.5875359177589417, 0.5538500547409058], "cos_innovation_negg": [0.5609899759292603, 0.5875359177589417, 0.5538500547409058], "cos_apical_negg": [0.5609899759292603, 0.5875359177589417, 0.5538500547409058], "cos_Ac_negg": [0.5609899759292603, 0.5875359177589417, 0.5538500547409058], "r_norm": [0.3110963702201843, 0.3101879358291626, 0.3150407075881958]}, {"step": 1950, "epoch": 4, "train_loss": 0.07906212657690048, "cos_r_negg": [0.556016206741333, 0.5785873532295227, 0.5427928566932678], "cos_innovation_negg": [0.556016206741333, 0.5785873532295227, 0.5427928566932678], "cos_apical_negg": [0.556016206741333, 0.5785873532295227, 0.5427928566932678], "cos_Ac_negg": [0.556016206741333, 0.5785873532295227, 0.5427928566932678], "r_norm": [0.21905165910720825, 0.219228595495224, 0.22225630283355713]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9659, "test_loss": 0.1229379430770874}, {"step": 2000, "epoch": 5, "train_loss": 0.025495946407318115, "cos_r_negg": [0.5521957278251648, 0.5752942562103271, 0.5501366257667542], "cos_innovation_negg": [0.5521957278251648, 0.5752942562103271, 0.5501366257667542], "cos_apical_negg": [0.5521957278251648, 0.5752942562103271, 0.5501366257667542], "cos_Ac_negg": [0.5521957278251648, 0.5752942562103271, 0.5501366257667542], "r_norm": [0.20675557851791382, 0.2066815197467804, 0.2105455994606018]}, {"step": 2050, "epoch": 5, "train_loss": 0.033479638397693634, "cos_r_negg": [0.5508012175559998, 0.5732553601264954, 0.5579526424407959], "cos_innovation_negg": [0.5508012175559998, 0.5732553601264954, 0.5579526424407959], "cos_apical_negg": [0.5508012175559998, 0.5732553601264954, 0.5579526424407959], "cos_Ac_negg": [0.5508012175559998, 0.5732553601264954, 0.5579526424407959], "r_norm": [0.20968316495418549, 0.2107418328523636, 0.21435150504112244]}, {"step": 2100, "epoch": 5, "train_loss": 0.035734161734580994, "cos_r_negg": [0.5821055173873901, 0.6019580364227295, 0.5754663944244385], "cos_innovation_negg": [0.5821055173873901, 0.6019580364227295, 0.5754663944244385], "cos_apical_negg": [0.5821055173873901, 0.6019580364227295, 0.5754663944244385], "cos_Ac_negg": [0.5821055173873901, 0.6019580364227295, 0.5754663944244385], "r_norm": [0.22412428259849548, 0.22490698099136353, 0.2282334417104721]}, {"step": 2150, "epoch": 5, "train_loss": 0.03620065748691559, "cos_r_negg": [0.5524879693984985, 0.5763399600982666, 0.5685200691223145], "cos_innovation_negg": [0.5524879693984985, 0.5763399600982666, 0.5685200691223145], "cos_apical_negg": [0.5524879693984985, 0.5763399600982666, 0.5685200691223145], "cos_Ac_negg": [0.5524879693984985, 0.5763399600982666, 0.5685200691223145], "r_norm": [0.2352142035961151, 0.23474735021591187, 0.2392353117465973]}, {"step": 2200, "epoch": 5, "train_loss": 0.025594597682356834, "cos_r_negg": [0.560375452041626, 0.5831714868545532, 0.5634292364120483], "cos_innovation_negg": [0.560375452041626, 0.5831714868545532, 0.5634292364120483], "cos_apical_negg": [0.560375452041626, 0.5831714868545532, 0.5634292364120483], "cos_Ac_negg": [0.560375452041626, 0.5831714868545532, 0.5634292364120483], "r_norm": [0.26695501804351807, 0.26720163226127625, 0.26951688528060913]}, {"step": 2250, "epoch": 5, "train_loss": 0.02437758445739746, "cos_r_negg": [0.5531279444694519, 0.5857134461402893, 0.5658981204032898], "cos_innovation_negg": [0.5531279444694519, 0.5857134461402893, 0.5658981204032898], "cos_apical_negg": [0.5531279444694519, 0.5857134461402893, 0.5658981204032898], "cos_Ac_negg": [0.5531279444694519, 0.5857134461402893, 0.5658981204032898], "r_norm": [0.26130881905555725, 0.26195865869522095, 0.2631542682647705]}, {"step": 2300, "epoch": 5, "train_loss": 0.025589298456907272, "cos_r_negg": [0.5679343938827515, 0.5880394577980042, 0.5790642499923706], "cos_innovation_negg": [0.5679343938827515, 0.5880394577980042, 0.5790642499923706], "cos_apical_negg": [0.5679343938827515, 0.5880394577980042, 0.5790642499923706], "cos_Ac_negg": [0.5679343938827515, 0.5880394577980042, 0.5790642499923706], "r_norm": [0.2239401787519455, 0.22328010201454163, 0.2239314764738083]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9687, "test_loss": 0.12023504390716552}, {"step": 2350, "epoch": 6, "train_loss": 0.010195987299084663, "cos_r_negg": [0.580016553401947, 0.5889356136322021, 0.5672563314437866], "cos_innovation_negg": [0.580016553401947, 0.5889356136322021, 0.5672563314437866], "cos_apical_negg": [0.580016553401947, 0.5889356136322021, 0.5672563314437866], "cos_Ac_negg": [0.580016553401947, 0.5889356136322021, 0.5672563314437866], "r_norm": [0.2353508621454239, 0.2366262823343277, 0.24040533602237701]}, {"step": 2400, "epoch": 6, "train_loss": 0.005657886154949665, "cos_r_negg": [0.5704793930053711, 0.593989908695221, 0.5747717022895813], "cos_innovation_negg": [0.5704793930053711, 0.593989908695221, 0.5747717022895813], "cos_apical_negg": [0.5704793930053711, 0.593989908695221, 0.5747717022895813], "cos_Ac_negg": [0.5704793930053711, 0.593989908695221, 0.5747717022895813], "r_norm": [0.20127269625663757, 0.201886385679245, 0.20541919767856598]}, {"step": 2450, "epoch": 6, "train_loss": 0.020389944314956665, "cos_r_negg": [0.5725216269493103, 0.596262514591217, 0.574916660785675], "cos_innovation_negg": [0.5725216269493103, 0.596262514591217, 0.574916660785675], "cos_apical_negg": [0.5725216269493103, 0.596262514591217, 0.574916660785675], "cos_Ac_negg": [0.5725216269493103, 0.596262514591217, 0.574916660785675], "r_norm": [0.22627118229866028, 0.2277900129556656, 0.2304127812385559]}, {"step": 2500, "epoch": 6, "train_loss": 0.027813628315925598, "cos_r_negg": [0.5746802687644958, 0.5943630337715149, 0.5795141458511353], "cos_innovation_negg": [0.5746802687644958, 0.5943630337715149, 0.5795141458511353], "cos_apical_negg": [0.5746802687644958, 0.5943630337715149, 0.5795141458511353], "cos_Ac_negg": [0.5746802687644958, 0.5943630337715149, 0.5795141458511353], "r_norm": [0.2193928062915802, 0.22106778621673584, 0.22385379672050476]}, {"step": 2550, "epoch": 6, "train_loss": 0.05354917049407959, "cos_r_negg": [0.5649705529212952, 0.5890545845031738, 0.5788935422897339], "cos_innovation_negg": [0.5649705529212952, 0.5890545845031738, 0.5788935422897339], "cos_apical_negg": [0.5649705529212952, 0.5890545845031738, 0.5788935422897339], "cos_Ac_negg": [0.5649705529212952, 0.5890545845031738, 0.5788935422897339], "r_norm": [0.20172613859176636, 0.2020355612039566, 0.20395049452781677]}, {"step": 2600, "epoch": 6, "train_loss": 0.02237768843770027, "cos_r_negg": [0.5712550282478333, 0.5940523743629456, 0.5894956588745117], "cos_innovation_negg": [0.5712550282478333, 0.5940523743629456, 0.5894956588745117], "cos_apical_negg": [0.5712550282478333, 0.5940523743629456, 0.5894956588745117], "cos_Ac_negg": [0.5712550282478333, 0.5940523743629456, 0.5894956588745117], "r_norm": [0.22900615632534027, 0.22782446444034576, 0.22920380532741547]}, {"step": 2650, "epoch": 6, "train_loss": 0.04657981917262077, "cos_r_negg": [0.5788272619247437, 0.6015475988388062, 0.5912551879882812], "cos_innovation_negg": [0.5788272619247437, 0.6015475988388062, 0.5912551879882812], "cos_apical_negg": [0.5788272619247437, 0.6015475988388062, 0.5912551879882812], "cos_Ac_negg": [0.5788272619247437, 0.6015475988388062, 0.5912551879882812], "r_norm": [0.19641482830047607, 0.19777441024780273, 0.20029395818710327]}, {"step": 2700, "epoch": 6, "train_loss": 0.003626564983278513, "cos_r_negg": [0.5680568218231201, 0.5988809466362, 0.6041505336761475], "cos_innovation_negg": [0.5680568218231201, 0.5988809466362, 0.6041505336761475], "cos_apical_negg": [0.5680568218231201, 0.5988809466362, 0.6041505336761475], "cos_Ac_negg": [0.5680568218231201, 0.5988809466362, 0.6041505336761475], "r_norm": [0.18219077587127686, 0.1832626909017563, 0.18720844388008118]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9694, "test_loss": 0.12068575963973999}, {"step": 2750, "epoch": 7, "train_loss": 0.003453271696344018, "cos_r_negg": [0.5821179747581482, 0.6088243722915649, 0.6056203842163086], "cos_innovation_negg": [0.5821179747581482, 0.6088243722915649, 0.6056203842163086], "cos_apical_negg": [0.5821179747581482, 0.6088243722915649, 0.6056203842163086], "cos_Ac_negg": [0.5821179747581482, 0.6088243722915649, 0.6056203842163086], "r_norm": [0.1696377992630005, 0.17014218866825104, 0.1748717576265335]}, {"step": 2800, "epoch": 7, "train_loss": 0.014115856029093266, "cos_r_negg": [0.5793547630310059, 0.6093785166740417, 0.6036314964294434], "cos_innovation_negg": [0.5793547630310059, 0.6093785166740417, 0.6036314964294434], "cos_apical_negg": [0.5793547630310059, 0.6093785166740417, 0.6036314964294434], "cos_Ac_negg": [0.5793547630310059, 0.6093785166740417, 0.6036314964294434], "r_norm": [0.18324440717697144, 0.1831023246049881, 0.18739549815654755]}, {"step": 2850, "epoch": 7, "train_loss": 0.005506793037056923, "cos_r_negg": [0.5799897909164429, 0.6048998832702637, 0.6089844703674316], "cos_innovation_negg": [0.5799897909164429, 0.6048998832702637, 0.6089844703674316], "cos_apical_negg": [0.5799897909164429, 0.6048998832702637, 0.6089844703674316], "cos_Ac_negg": [0.5799897909164429, 0.6048998832702637, 0.6089844703674316], "r_norm": [0.18239723145961761, 0.1838771104812622, 0.18733710050582886]}, {"step": 2900, "epoch": 7, "train_loss": 0.01831877790391445, "cos_r_negg": [0.580494225025177, 0.6074983477592468, 0.6150795221328735], "cos_innovation_negg": [0.580494225025177, 0.6074983477592468, 0.6150795221328735], "cos_apical_negg": [0.580494225025177, 0.6074983477592468, 0.6150795221328735], "cos_Ac_negg": [0.580494225025177, 0.6074983477592468, 0.6150795221328735], "r_norm": [0.1610778272151947, 0.16192734241485596, 0.1644197553396225]}, {"step": 2950, "epoch": 7, "train_loss": 0.003671329002827406, "cos_r_negg": [0.5746171474456787, 0.6073042750358582, 0.6119168996810913], "cos_innovation_negg": [0.5746171474456787, 0.6073042750358582, 0.6119168996810913], "cos_apical_negg": [0.5746171474456787, 0.6073042750358582, 0.6119168996810913], "cos_Ac_negg": [0.5746171474456787, 0.6073042750358582, 0.6119168996810913], "r_norm": [0.18937592208385468, 0.190243661403656, 0.19300894439220428]}, {"step": 3000, "epoch": 7, "train_loss": 0.01588994823396206, "cos_r_negg": [0.5889807939529419, 0.6140233278274536, 0.6150346994400024], "cos_innovation_negg": [0.5889807939529419, 0.6140233278274536, 0.6150346994400024], "cos_apical_negg": [0.5889807939529419, 0.6140233278274536, 0.6150346994400024], "cos_Ac_negg": [0.5889807939529419, 0.6140233278274536, 0.6150346994400024], "r_norm": [0.19920217990875244, 0.1994800567626953, 0.20139873027801514]}, {"step": 3050, "epoch": 7, "train_loss": 0.005256535019725561, "cos_r_negg": [0.5845313668251038, 0.6172623038291931, 0.6184163093566895], "cos_innovation_negg": [0.5845313668251038, 0.6172623038291931, 0.6184163093566895], "cos_apical_negg": [0.5845313668251038, 0.6172623038291931, 0.6184163093566895], "cos_Ac_negg": [0.5845313668251038, 0.6172623038291931, 0.6184163093566895], "r_norm": [0.18376994132995605, 0.18464341759681702, 0.18719390034675598]}, {"step": 3100, "epoch": 7, "train_loss": 0.0013961158692836761, "cos_r_negg": [0.5833116769790649, 0.61182701587677, 0.6187093257904053], "cos_innovation_negg": [0.5833116769790649, 0.61182701587677, 0.6187093257904053], "cos_apical_negg": [0.5833116769790649, 0.61182701587677, 0.6187093257904053], "cos_Ac_negg": [0.5833116769790649, 0.61182701587677, 0.6187093257904053], "r_norm": [0.15723830461502075, 0.1570248007774353, 0.16056130826473236]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.97, "test_loss": 0.12961395721435548}, {"step": 3150, "epoch": 8, "train_loss": 0.0021287165582180023, "cos_r_negg": [0.5797766447067261, 0.6182849407196045, 0.6204319596290588], "cos_innovation_negg": [0.5797766447067261, 0.6182849407196045, 0.6204319596290588], "cos_apical_negg": [0.5797766447067261, 0.6182849407196045, 0.6204319596290588], "cos_Ac_negg": [0.5797766447067261, 0.6182849407196045, 0.6204319596290588], "r_norm": [0.20185869932174683, 0.20194733142852783, 0.20272590219974518]}, {"step": 3200, "epoch": 8, "train_loss": 0.02433200180530548, "cos_r_negg": [0.5772427320480347, 0.616121768951416, 0.6229267120361328], "cos_innovation_negg": [0.5772427320480347, 0.616121768951416, 0.6229267120361328], "cos_apical_negg": [0.5772427320480347, 0.616121768951416, 0.6229267120361328], "cos_Ac_negg": [0.5772427320480347, 0.616121768951416, 0.6229267120361328], "r_norm": [0.1926865130662918, 0.19218260049819946, 0.1948787420988083]}, {"step": 3250, "epoch": 8, "train_loss": 0.005649031139910221, "cos_r_negg": [0.57783442735672, 0.6194196343421936, 0.6247861385345459], "cos_innovation_negg": [0.57783442735672, 0.6194196343421936, 0.6247861385345459], "cos_apical_negg": [0.57783442735672, 0.6194196343421936, 0.6247861385345459], "cos_Ac_negg": [0.57783442735672, 0.6194196343421936, 0.6247861385345459], "r_norm": [0.1985979825258255, 0.1982056349515915, 0.20070844888687134]}, {"step": 3300, "epoch": 8, "train_loss": 0.009066684171557426, "cos_r_negg": [0.5764681100845337, 0.6184250116348267, 0.6216678023338318], "cos_innovation_negg": [0.5764681100845337, 0.6184250116348267, 0.6216678023338318], "cos_apical_negg": [0.5764681100845337, 0.6184250116348267, 0.6216678023338318], "cos_Ac_negg": [0.5764681100845337, 0.6184250116348267, 0.6216678023338318], "r_norm": [0.19004544615745544, 0.19049635529518127, 0.19161690771579742]}, {"step": 3350, "epoch": 8, "train_loss": 0.012781675904989243, "cos_r_negg": [0.5774061679840088, 0.6193328499794006, 0.6248455047607422], "cos_innovation_negg": [0.5774061679840088, 0.6193328499794006, 0.6248455047607422], "cos_apical_negg": [0.5774061679840088, 0.6193328499794006, 0.6248455047607422], "cos_Ac_negg": [0.5774061679840088, 0.6193328499794006, 0.6248455047607422], "r_norm": [0.1940985918045044, 0.195678249001503, 0.19837820529937744]}, {"step": 3400, "epoch": 8, "train_loss": 0.013611163944005966, "cos_r_negg": [0.5776745080947876, 0.6177480220794678, 0.6247376203536987], "cos_innovation_negg": [0.5776745080947876, 0.6177480220794678, 0.6247376203536987], "cos_apical_negg": [0.5776745080947876, 0.6177480220794678, 0.6247376203536987], "cos_Ac_negg": [0.5776745080947876, 0.6177480220794678, 0.6247376203536987], "r_norm": [0.19726720452308655, 0.1977410614490509, 0.20152604579925537]}, {"step": 3450, "epoch": 8, "train_loss": 0.017978962510824203, "cos_r_negg": [0.5828019380569458, 0.622667133808136, 0.6316971182823181], "cos_innovation_negg": [0.5828019380569458, 0.622667133808136, 0.6316971182823181], "cos_apical_negg": [0.5828019380569458, 0.622667133808136, 0.6316971182823181], "cos_Ac_negg": [0.5828019380569458, 0.622667133808136, 0.6316971182823181], "r_norm": [0.18927831947803497, 0.1903052031993866, 0.19371889531612396]}, {"step": 3500, "epoch": 8, "train_loss": 0.025722026824951172, "cos_r_negg": [0.5802310109138489, 0.6198745369911194, 0.6321008205413818], "cos_innovation_negg": [0.5802310109138489, 0.6198745369911194, 0.6321008205413818], "cos_apical_negg": [0.5802310109138489, 0.6198745369911194, 0.6321008205413818], "cos_Ac_negg": [0.5802310109138489, 0.6198745369911194, 0.6321008205413818], "r_norm": [0.21116656064987183, 0.21123838424682617, 0.21507088840007782]}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9716, "test_loss": 0.13078003787994386}, {"step": 3550, "epoch": 9, "train_loss": 0.0072563411667943, "cos_r_negg": [0.582394540309906, 0.6242104172706604, 0.63521409034729], "cos_innovation_negg": [0.582394540309906, 0.6242104172706604, 0.63521409034729], "cos_apical_negg": [0.582394540309906, 0.6242104172706604, 0.63521409034729], "cos_Ac_negg": [0.582394540309906, 0.6242104172706604, 0.63521409034729], "r_norm": [0.18871960043907166, 0.18787705898284912, 0.19132477045059204]}, {"step": 3600, "epoch": 9, "train_loss": 0.0017336413729935884, "cos_r_negg": [0.5780080556869507, 0.6216466426849365, 0.6344834566116333], "cos_innovation_negg": [0.5780080556869507, 0.6216466426849365, 0.6344834566116333], "cos_apical_negg": [0.5780080556869507, 0.6216466426849365, 0.6344834566116333], "cos_Ac_negg": [0.5780080556869507, 0.6216466426849365, 0.6344834566116333], "r_norm": [0.17696890234947205, 0.1761516034603119, 0.17876246571540833]}, {"step": 3650, "epoch": 9, "train_loss": 0.004210930783301592, "cos_r_negg": [0.5840017795562744, 0.6235603094100952, 0.6383466720581055], "cos_innovation_negg": [0.5840017795562744, 0.6235603094100952, 0.6383466720581055], "cos_apical_negg": [0.5840017795562744, 0.6235603094100952, 0.6383466720581055], "cos_Ac_negg": [0.5840017795562744, 0.6235603094100952, 0.6383466720581055], "r_norm": [0.18332073092460632, 0.1823318600654602, 0.18429310619831085]}, {"step": 3700, "epoch": 9, "train_loss": 0.011482157744467258, "cos_r_negg": [0.5770446062088013, 0.6187848448753357, 0.6345071792602539], "cos_innovation_negg": [0.5770446062088013, 0.6187848448753357, 0.6345071792602539], "cos_apical_negg": [0.5770446062088013, 0.6187848448753357, 0.6345071792602539], "cos_Ac_negg": [0.5770446062088013, 0.6187848448753357, 0.6345071792602539], "r_norm": [0.17805451154708862, 0.17794623970985413, 0.18069742619991302]}, {"step": 3750, "epoch": 9, "train_loss": 0.0026844998355954885, "cos_r_negg": [0.5807583332061768, 0.622568666934967, 0.6347371339797974], "cos_innovation_negg": [0.5807583332061768, 0.622568666934967, 0.6347371339797974], "cos_apical_negg": [0.5807583332061768, 0.622568666934967, 0.6347371339797974], "cos_Ac_negg": [0.5807583332061768, 0.622568666934967, 0.6347371339797974], "r_norm": [0.18593746423721313, 0.18578273057937622, 0.1882787048816681]}, {"step": 3800, "epoch": 9, "train_loss": 0.0015858508413657546, "cos_r_negg": [0.5758232474327087, 0.6255663633346558, 0.638684868812561], "cos_innovation_negg": [0.5758232474327087, 0.6255663633346558, 0.638684868812561], "cos_apical_negg": [0.5758232474327087, 0.6255663633346558, 0.638684868812561], "cos_Ac_negg": [0.5758232474327087, 0.6255663633346558, 0.638684868812561], "r_norm": [0.18229137361049652, 0.18143439292907715, 0.18379969894886017]}, {"step": 3850, "epoch": 9, "train_loss": 0.007211725227534771, "cos_r_negg": [0.5802623629570007, 0.6275233030319214, 0.6379368305206299], "cos_innovation_negg": [0.5802623629570007, 0.6275233030319214, 0.6379368305206299], "cos_apical_negg": [0.5802623629570007, 0.6275233030319214, 0.6379368305206299], "cos_Ac_negg": [0.5802623629570007, 0.6275233030319214, 0.6379368305206299], "r_norm": [0.19035634398460388, 0.1898869276046753, 0.1922195702791214]}, {"step": 3900, "epoch": 9, "train_loss": 0.0065520345233380795, "cos_r_negg": [0.5796908140182495, 0.6270134449005127, 0.6414741277694702], "cos_innovation_negg": [0.5796908140182495, 0.6270134449005127, 0.6414741277694702], "cos_apical_negg": [0.5796908140182495, 0.6270134449005127, 0.6414741277694702], "cos_Ac_negg": [0.5796908140182495, 0.6270134449005127, 0.6414741277694702], "r_norm": [0.16825298964977264, 0.1683555543422699, 0.17128270864486694]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9719, "test_loss": 0.13426431455612184}, {"step": 3950, "epoch": 10, "train_loss": 0.0009596800664439797, "cos_r_negg": [0.5793710947036743, 0.6249526739120483, 0.643487811088562], "cos_innovation_negg": [0.5793710947036743, 0.6249526739120483, 0.643487811088562], "cos_apical_negg": [0.5793710947036743, 0.6249526739120483, 0.643487811088562], "cos_Ac_negg": [0.5793710947036743, 0.6249526739120483, 0.643487811088562], "r_norm": [0.17605698108673096, 0.175759956240654, 0.17845913767814636]}, {"step": 4000, "epoch": 10, "train_loss": 0.001352860825136304, "cos_r_negg": [0.5815757513046265, 0.6274793148040771, 0.6431308388710022], "cos_innovation_negg": [0.5815757513046265, 0.6274793148040771, 0.6431308388710022], "cos_apical_negg": [0.5815757513046265, 0.6274793148040771, 0.6431308388710022], "cos_Ac_negg": [0.5815757513046265, 0.6274793148040771, 0.6431308388710022], "r_norm": [0.1790170967578888, 0.1785300374031067, 0.18065765500068665]}, {"step": 4050, "epoch": 10, "train_loss": 0.0012009716592729092, "cos_r_negg": [0.5770502090454102, 0.6242483854293823, 0.6432619094848633], "cos_innovation_negg": [0.5770502090454102, 0.6242483854293823, 0.6432619094848633], "cos_apical_negg": [0.5770502090454102, 0.6242483854293823, 0.6432619094848633], "cos_Ac_negg": [0.5770502090454102, 0.6242483854293823, 0.6432619094848633], "r_norm": [0.16967767477035522, 0.16950900852680206, 0.17225182056427002]}, {"step": 4100, "epoch": 10, "train_loss": 0.0010594265768304467, "cos_r_negg": [0.5826135873794556, 0.6273981332778931, 0.6425342559814453], "cos_innovation_negg": [0.5826135873794556, 0.6273981332778931, 0.6425342559814453], "cos_apical_negg": [0.5826135873794556, 0.6273981332778931, 0.6425342559814453], "cos_Ac_negg": [0.5826135873794556, 0.6273981332778931, 0.6425342559814453], "r_norm": [0.1731465607881546, 0.1730847954750061, 0.17520514130592346]}, {"step": 4150, "epoch": 10, "train_loss": 0.0007644288707524538, "cos_r_negg": [0.5820911526679993, 0.6286026239395142, 0.6433957815170288], "cos_innovation_negg": [0.5820911526679993, 0.6286026239395142, 0.6433957815170288], "cos_apical_negg": [0.5820911526679993, 0.6286026239395142, 0.6433957815170288], "cos_Ac_negg": [0.5820911526679993, 0.6286026239395142, 0.6433957815170288], "r_norm": [0.16967862844467163, 0.16986632347106934, 0.1730210781097412]}, {"step": 4200, "epoch": 10, "train_loss": 0.001654764055274427, "cos_r_negg": [0.58017897605896, 0.6296882629394531, 0.6420108675956726], "cos_innovation_negg": [0.58017897605896, 0.6296882629394531, 0.6420108675956726], "cos_apical_negg": [0.58017897605896, 0.6296882629394531, 0.6420108675956726], "cos_Ac_negg": [0.58017897605896, 0.6296882629394531, 0.6420108675956726], "r_norm": [0.15985962748527527, 0.1601172238588333, 0.16318929195404053]}, {"step": 4250, "epoch": 10, "train_loss": 0.005034513305872679, "cos_r_negg": [0.5741692185401917, 0.6284055709838867, 0.6436196565628052], "cos_innovation_negg": [0.5741692185401917, 0.6284055709838867, 0.6436196565628052], "cos_apical_negg": [0.5741692185401917, 0.6284055709838867, 0.6436196565628052], "cos_Ac_negg": [0.5741692185401917, 0.6284055709838867, 0.6436196565628052], "r_norm": [0.1524534821510315, 0.15253379940986633, 0.15595528483390808]}, {"step": 4300, "epoch": 10, "train_loss": 0.010711022652685642, "cos_r_negg": [0.5737922787666321, 0.6265794634819031, 0.6450129747390747], "cos_innovation_negg": [0.5737922787666321, 0.6265794634819031, 0.6450129747390747], "cos_apical_negg": [0.5737922787666321, 0.6265794634819031, 0.6450129747390747], "cos_Ac_negg": [0.5737922787666321, 0.6265794634819031, 0.6450129747390747], "r_norm": [0.15216845273971558, 0.1518905758857727, 0.15535087883472443]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9721, "test_loss": 0.13763998260498048}, {"step": 4350, "epoch": 11, "train_loss": 0.002100103534758091, "cos_r_negg": [0.5754216909408569, 0.6280955076217651, 0.6451839208602905], "cos_innovation_negg": [0.5754216909408569, 0.6280955076217651, 0.6451839208602905], "cos_apical_negg": [0.5754216909408569, 0.6280955076217651, 0.6451839208602905], "cos_Ac_negg": [0.5754216909408569, 0.6280955076217651, 0.6451839208602905], "r_norm": [0.16137200593948364, 0.16126926243305206, 0.16429950296878815]}, {"step": 4400, "epoch": 11, "train_loss": 0.0010687396861612797, "cos_r_negg": [0.5712452530860901, 0.6285752058029175, 0.6459286212921143], "cos_innovation_negg": [0.5712452530860901, 0.6285752058029175, 0.6459286212921143], "cos_apical_negg": [0.5712452530860901, 0.6285752058029175, 0.6459286212921143], "cos_Ac_negg": [0.5712452530860901, 0.6285752058029175, 0.6459286212921143], "r_norm": [0.1615949422121048, 0.16153767704963684, 0.16466227173805237]}, {"step": 4450, "epoch": 11, "train_loss": 0.002542033325880766, "cos_r_negg": [0.5726916193962097, 0.6308110952377319, 0.6460199952125549], "cos_innovation_negg": [0.5726916193962097, 0.6308110952377319, 0.6460199952125549], "cos_apical_negg": [0.5726916193962097, 0.6308110952377319, 0.6460199952125549], "cos_Ac_negg": [0.5726916193962097, 0.6308110952377319, 0.6460199952125549], "r_norm": [0.16201616823673248, 0.16182206571102142, 0.16487492620944977]}, {"step": 4500, "epoch": 11, "train_loss": 0.00023055251222103834, "cos_r_negg": [0.5731532573699951, 0.629715621471405, 0.6451817750930786], "cos_innovation_negg": [0.5731532573699951, 0.629715621471405, 0.6451817750930786], "cos_apical_negg": [0.5731532573699951, 0.629715621471405, 0.6451817750930786], "cos_Ac_negg": [0.5731532573699951, 0.629715621471405, 0.6451817750930786], "r_norm": [0.16165751218795776, 0.16142915189266205, 0.1645810753107071]}, {"step": 4550, "epoch": 11, "train_loss": 0.0022498909384012222, "cos_r_negg": [0.5707855224609375, 0.6291557550430298, 0.646937370300293], "cos_innovation_negg": [0.5707855224609375, 0.6291557550430298, 0.646937370300293], "cos_apical_negg": [0.5707855224609375, 0.6291557550430298, 0.646937370300293], "cos_Ac_negg": [0.5707855224609375, 0.6291557550430298, 0.646937370300293], "r_norm": [0.16657891869544983, 0.1659044772386551, 0.1686849743127823]}, {"step": 4600, "epoch": 11, "train_loss": 0.00040937442099675536, "cos_r_negg": [0.5753434896469116, 0.6295727491378784, 0.6466732025146484], "cos_innovation_negg": [0.5753434896469116, 0.6295727491378784, 0.6466732025146484], "cos_apical_negg": [0.5753434896469116, 0.6295727491378784, 0.6466732025146484], "cos_Ac_negg": [0.5753434896469116, 0.6295727491378784, 0.6466732025146484], "r_norm": [0.17542795836925507, 0.17505250871181488, 0.17761683464050293]}, {"step": 4650, "epoch": 11, "train_loss": 0.0004352562827989459, "cos_r_negg": [0.5791181325912476, 0.6309177875518799, 0.6481046676635742], "cos_innovation_negg": [0.5791181325912476, 0.6309177875518799, 0.6481046676635742], "cos_apical_negg": [0.5791181325912476, 0.6309177875518799, 0.6481046676635742], "cos_Ac_negg": [0.5791181325912476, 0.6309177875518799, 0.6481046676635742], "r_norm": [0.16686901450157166, 0.16627250611782074, 0.16913756728172302]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9724, "test_loss": 0.14070229959487915}, {"step": 4700, "epoch": 12, "train_loss": 0.0014199765864759684, "cos_r_negg": [0.5782716274261475, 0.6307209730148315, 0.6481973528862], "cos_innovation_negg": [0.5782716274261475, 0.6307209730148315, 0.6481973528862], "cos_apical_negg": [0.5782716274261475, 0.6307209730148315, 0.6481973528862], "cos_Ac_negg": [0.5782716274261475, 0.6307209730148315, 0.6481973528862], "r_norm": [0.1570122390985489, 0.15675407648086548, 0.16001483798027039]}, {"step": 4750, "epoch": 12, "train_loss": 0.0017644870094954967, "cos_r_negg": [0.5781770944595337, 0.6324417591094971, 0.6488444805145264], "cos_innovation_negg": [0.5781770944595337, 0.6324417591094971, 0.6488444805145264], "cos_apical_negg": [0.5781770944595337, 0.6324417591094971, 0.6488444805145264], "cos_Ac_negg": [0.5781770944595337, 0.6324417591094971, 0.6488444805145264], "r_norm": [0.15826106071472168, 0.15848734974861145, 0.16171009838581085]}, {"step": 4800, "epoch": 12, "train_loss": 0.00027746264822781086, "cos_r_negg": [0.576261043548584, 0.6313995718955994, 0.6482354402542114], "cos_innovation_negg": [0.576261043548584, 0.6313995718955994, 0.6482354402542114], "cos_apical_negg": [0.576261043548584, 0.6313995718955994, 0.6482354402542114], "cos_Ac_negg": [0.576261043548584, 0.6313995718955994, 0.6482354402542114], "r_norm": [0.1658482700586319, 0.1661399006843567, 0.16905994713306427]}, {"step": 4850, "epoch": 12, "train_loss": 0.0004152090987190604, "cos_r_negg": [0.5746157169342041, 0.6324016451835632, 0.6483161449432373], "cos_innovation_negg": [0.5746157169342041, 0.6324016451835632, 0.6483161449432373], "cos_apical_negg": [0.5746157169342041, 0.6324016451835632, 0.6483161449432373], "cos_Ac_negg": [0.5746157169342041, 0.6324016451835632, 0.6483161449432373], "r_norm": [0.1678943783044815, 0.16777901351451874, 0.1707860827445984]}, {"step": 4900, "epoch": 12, "train_loss": 0.0007552538299933076, "cos_r_negg": [0.5714161396026611, 0.6302602291107178, 0.6481420993804932], "cos_innovation_negg": [0.5714161396026611, 0.6302602291107178, 0.6481420993804932], "cos_apical_negg": [0.5714161396026611, 0.6302602291107178, 0.6481420993804932], "cos_Ac_negg": [0.5714161396026611, 0.6302602291107178, 0.6481420993804932], "r_norm": [0.16494962573051453, 0.16461773216724396, 0.1675436943769455]}, {"step": 4950, "epoch": 12, "train_loss": 0.0005488710012286901, "cos_r_negg": [0.5725513696670532, 0.631328284740448, 0.6489154696464539], "cos_innovation_negg": [0.5725513696670532, 0.631328284740448, 0.6489154696464539], "cos_apical_negg": [0.5725513696670532, 0.631328284740448, 0.6489154696464539], "cos_Ac_negg": [0.5725513696670532, 0.631328284740448, 0.6489154696464539], "r_norm": [0.16278904676437378, 0.1627579778432846, 0.16543978452682495]}, {"step": 5000, "epoch": 12, "train_loss": 0.00024091819068416953, "cos_r_negg": [0.5747544765472412, 0.6309801936149597, 0.6496747732162476], "cos_innovation_negg": [0.5747544765472412, 0.6309801936149597, 0.6496747732162476], "cos_apical_negg": [0.5747544765472412, 0.6309801936149597, 0.6496747732162476], "cos_Ac_negg": [0.5747544765472412, 0.6309801936149597, 0.6496747732162476], "r_norm": [0.16480715572834015, 0.16468696296215057, 0.1674092710018158]}, {"step": 5050, "epoch": 12, "train_loss": 0.0005048258462920785, "cos_r_negg": [0.5769394636154175, 0.6302328705787659, 0.6506783366203308], "cos_innovation_negg": [0.5769394636154175, 0.6302328705787659, 0.6506783366203308], "cos_apical_negg": [0.5769394636154175, 0.6302328705787659, 0.6506783366203308], "cos_Ac_negg": [0.5769394636154175, 0.6302328705787659, 0.6506783366203308], "r_norm": [0.17127138376235962, 0.1708337664604187, 0.17316381633281708]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9717, "test_loss": 0.14478787775039673}, {"step": 5100, "epoch": 13, "train_loss": 0.0003445396141614765, "cos_r_negg": [0.5748473405838013, 0.63182133436203, 0.6506901979446411], "cos_innovation_negg": [0.5748473405838013, 0.63182133436203, 0.6506901979446411], "cos_apical_negg": [0.5748473405838013, 0.63182133436203, 0.6506901979446411], "cos_Ac_negg": [0.5748473405838013, 0.63182133436203, 0.6506901979446411], "r_norm": [0.17519091069698334, 0.17459072172641754, 0.17682568728923798]}, {"step": 5150, "epoch": 13, "train_loss": 0.0005578401032835245, "cos_r_negg": [0.5765137076377869, 0.6332394480705261, 0.651635468006134], "cos_innovation_negg": [0.5765137076377869, 0.6332394480705261, 0.651635468006134], "cos_apical_negg": [0.5765137076377869, 0.6332394480705261, 0.651635468006134], "cos_Ac_negg": [0.5765137076377869, 0.6332394480705261, 0.651635468006134], "r_norm": [0.16716647148132324, 0.16691303253173828, 0.1694052517414093]}, {"step": 5200, "epoch": 13, "train_loss": 0.0009801641572266817, "cos_r_negg": [0.5752623081207275, 0.6330417394638062, 0.6514869928359985], "cos_innovation_negg": [0.5752623081207275, 0.6330417394638062, 0.6514869928359985], "cos_apical_negg": [0.5752623081207275, 0.6330417394638062, 0.6514869928359985], "cos_Ac_negg": [0.5752623081207275, 0.6330417394638062, 0.6514869928359985], "r_norm": [0.16601960361003876, 0.1659834086894989, 0.16873764991760254]}, {"step": 5250, "epoch": 13, "train_loss": 0.00042195903370156884, "cos_r_negg": [0.5740411281585693, 0.6325175762176514, 0.6527026295661926], "cos_innovation_negg": [0.5740411281585693, 0.6325175762176514, 0.6527026295661926], "cos_apical_negg": [0.5740411281585693, 0.6325175762176514, 0.6527026295661926], "cos_Ac_negg": [0.5740411281585693, 0.6325175762176514, 0.6527026295661926], "r_norm": [0.1626453846693039, 0.1628461480140686, 0.165805846452713]}, {"step": 5300, "epoch": 13, "train_loss": 0.0005502301501110196, "cos_r_negg": [0.5735984444618225, 0.631951630115509, 0.6515814065933228], "cos_innovation_negg": [0.5735984444618225, 0.631951630115509, 0.6515814065933228], "cos_apical_negg": [0.5735984444618225, 0.631951630115509, 0.6515814065933228], "cos_Ac_negg": [0.5735984444618225, 0.631951630115509, 0.6515814065933228], "r_norm": [0.16218626499176025, 0.16200780868530273, 0.16496418416500092]}, {"step": 5350, "epoch": 13, "train_loss": 0.0011079205432906747, "cos_r_negg": [0.575512170791626, 0.6329067945480347, 0.6518474817276001], "cos_innovation_negg": [0.575512170791626, 0.6329067945480347, 0.6518474817276001], "cos_apical_negg": [0.575512170791626, 0.6329067945480347, 0.6518474817276001], "cos_Ac_negg": [0.575512170791626, 0.6329067945480347, 0.6518474817276001], "r_norm": [0.16268901526927948, 0.1624549925327301, 0.16515180468559265]}, {"step": 5400, "epoch": 13, "train_loss": 0.0012177620083093643, "cos_r_negg": [0.5748374462127686, 0.6333304643630981, 0.6528788208961487], "cos_innovation_negg": [0.5748374462127686, 0.6333304643630981, 0.6528788208961487], "cos_apical_negg": [0.5748374462127686, 0.6333304643630981, 0.6528788208961487], "cos_Ac_negg": [0.5748374462127686, 0.6333304643630981, 0.6528788208961487], "r_norm": [0.16146960854530334, 0.16127242147922516, 0.16414371132850647]}, {"step": 5450, "epoch": 13, "train_loss": 0.00034425538615323603, "cos_r_negg": [0.5739434957504272, 0.6337035894393921, 0.6526656150817871], "cos_innovation_negg": [0.5739434957504272, 0.6337035894393921, 0.6526656150817871], "cos_apical_negg": [0.5739434957504272, 0.6337035894393921, 0.6526656150817871], "cos_Ac_negg": [0.5739434957504272, 0.6337035894393921, 0.6526656150817871], "r_norm": [0.16002890467643738, 0.16020214557647705, 0.1634407341480255]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9728, "test_loss": 0.14756818046569825}, {"step": 5500, "epoch": 14, "train_loss": 0.0004224753938615322, "cos_r_negg": [0.5740289688110352, 0.6334036588668823, 0.6529339551925659], "cos_innovation_negg": [0.5740289688110352, 0.6334036588668823, 0.6529339551925659], "cos_apical_negg": [0.5740289688110352, 0.6334036588668823, 0.6529339551925659], "cos_Ac_negg": [0.5740289688110352, 0.6334036588668823, 0.6529339551925659], "r_norm": [0.16378869116306305, 0.16404935717582703, 0.1671530157327652]}, {"step": 5550, "epoch": 14, "train_loss": 0.00019850602257065475, "cos_r_negg": [0.5725598335266113, 0.6335716247558594, 0.6522932052612305], "cos_innovation_negg": [0.5725598335266113, 0.6335716247558594, 0.6522932052612305], "cos_apical_negg": [0.5725598335266113, 0.6335716247558594, 0.6522932052612305], "cos_Ac_negg": [0.5725598335266113, 0.6335716247558594, 0.6522932052612305], "r_norm": [0.1646849662065506, 0.16488884389400482, 0.1680004596710205]}, {"step": 5600, "epoch": 14, "train_loss": 0.0012179003097116947, "cos_r_negg": [0.5727795362472534, 0.6328481435775757, 0.6526288986206055], "cos_innovation_negg": [0.5727795362472534, 0.6328481435775757, 0.6526288986206055], "cos_apical_negg": [0.5727795362472534, 0.6328481435775757, 0.6526288986206055], "cos_Ac_negg": [0.5727795362472534, 0.6328481435775757, 0.6526288986206055], "r_norm": [0.16751298308372498, 0.16775381565093994, 0.17058494687080383]}, {"step": 5650, "epoch": 14, "train_loss": 0.0006461373413912952, "cos_r_negg": [0.5728119611740112, 0.6327627897262573, 0.6521763801574707], "cos_innovation_negg": [0.5728119611740112, 0.6327627897262573, 0.6521763801574707], "cos_apical_negg": [0.5728119611740112, 0.6327627897262573, 0.6521763801574707], "cos_Ac_negg": [0.5728119611740112, 0.6327627897262573, 0.6521763801574707], "r_norm": [0.1645781397819519, 0.16490355134010315, 0.16781875491142273]}, {"step": 5700, "epoch": 14, "train_loss": 0.00015112292021512985, "cos_r_negg": [0.5717822313308716, 0.6335359811782837, 0.6527776718139648], "cos_innovation_negg": [0.5717822313308716, 0.6335359811782837, 0.6527776718139648], "cos_apical_negg": [0.5717822313308716, 0.6335359811782837, 0.6527776718139648], "cos_Ac_negg": [0.5717822313308716, 0.6335359811782837, 0.6527776718139648], "r_norm": [0.163626030087471, 0.16386209428310394, 0.16670677065849304]}, {"step": 5750, "epoch": 14, "train_loss": 0.0004824709612876177, "cos_r_negg": [0.5710581541061401, 0.6338963508605957, 0.6552304029464722], "cos_innovation_negg": [0.5710581541061401, 0.6338963508605957, 0.6552304029464722], "cos_apical_negg": [0.5710581541061401, 0.6338963508605957, 0.6552304029464722], "cos_Ac_negg": [0.5710581541061401, 0.6338963508605957, 0.6552304029464722], "r_norm": [0.16106702387332916, 0.1615605354309082, 0.16439606249332428]}, {"step": 5800, "epoch": 14, "train_loss": 0.0017331193666905165, "cos_r_negg": [0.5708421468734741, 0.6338798403739929, 0.6543550491333008], "cos_innovation_negg": [0.5708421468734741, 0.6338798403739929, 0.6543550491333008], "cos_apical_negg": [0.5708421468734741, 0.6338798403739929, 0.6543550491333008], "cos_Ac_negg": [0.5708421468734741, 0.6338798403739929, 0.6543550491333008], "r_norm": [0.16586796939373016, 0.16616064310073853, 0.1688176840543747]}, {"step": 5850, "epoch": 14, "train_loss": 0.00046079789171926677, "cos_r_negg": [0.5752074718475342, 0.6320285201072693, 0.6525647640228271], "cos_innovation_negg": [0.5752074718475342, 0.6320285201072693, 0.6525647640228271], "cos_apical_negg": [0.5752074718475342, 0.6320285201072693, 0.6525647640228271], "cos_Ac_negg": [0.5752074718475342, 0.6320285201072693, 0.6525647640228271], "r_norm": [0.16546961665153503, 0.1657031774520874, 0.16845229268074036]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9727, "test_loss": 0.14990826272964478}, {"step": 5900, "epoch": 15, "train_loss": 0.00048335635801777244, "cos_r_negg": [0.573550283908844, 0.633561372756958, 0.654201090335846], "cos_innovation_negg": [0.573550283908844, 0.633561372756958, 0.654201090335846], "cos_apical_negg": [0.573550283908844, 0.633561372756958, 0.654201090335846], "cos_Ac_negg": [0.573550283908844, 0.633561372756958, 0.654201090335846], "r_norm": [0.16834810376167297, 0.16832417249679565, 0.17098209261894226]}, {"step": 5950, "epoch": 15, "train_loss": 0.00032604881562292576, "cos_r_negg": [0.5724385380744934, 0.6333562135696411, 0.6542192697525024], "cos_innovation_negg": [0.5724385380744934, 0.6333562135696411, 0.6542192697525024], "cos_apical_negg": [0.5724385380744934, 0.6333562135696411, 0.6542192697525024], "cos_Ac_negg": [0.5724385380744934, 0.6333562135696411, 0.6542192697525024], "r_norm": [0.1644570231437683, 0.16463926434516907, 0.16750138998031616]}, {"step": 6000, "epoch": 15, "train_loss": 0.0005397888598963618, "cos_r_negg": [0.5732101798057556, 0.6326330900192261, 0.6525570750236511], "cos_innovation_negg": [0.5732101798057556, 0.6326330900192261, 0.6525570750236511], "cos_apical_negg": [0.5732101798057556, 0.6326330900192261, 0.6525570750236511], "cos_Ac_negg": [0.5732101798057556, 0.6326330900192261, 0.6525570750236511], "r_norm": [0.1654687076807022, 0.1655559241771698, 0.1684839129447937]}, {"step": 6050, "epoch": 15, "train_loss": 0.0029816809110343456, "cos_r_negg": [0.571812629699707, 0.6328395009040833, 0.653708279132843], "cos_innovation_negg": [0.571812629699707, 0.6328395009040833, 0.653708279132843], "cos_apical_negg": [0.571812629699707, 0.6328395009040833, 0.653708279132843], "cos_Ac_negg": [0.571812629699707, 0.6328395009040833, 0.653708279132843], "r_norm": [0.1664140820503235, 0.1665171980857849, 0.16939733922481537]}, {"step": 6100, "epoch": 15, "train_loss": 0.0011848782887682319, "cos_r_negg": [0.57236248254776, 0.6312166452407837, 0.6517926454544067], "cos_innovation_negg": [0.57236248254776, 0.6312166452407837, 0.6517926454544067], "cos_apical_negg": [0.57236248254776, 0.6312166452407837, 0.6517926454544067], "cos_Ac_negg": [0.57236248254776, 0.6312166452407837, 0.6517926454544067], "r_norm": [0.1690715253353119, 0.16904309391975403, 0.1717344969511032]}, {"step": 6150, "epoch": 15, "train_loss": 0.0004526081611402333, "cos_r_negg": [0.573922872543335, 0.6326964497566223, 0.6536688804626465], "cos_innovation_negg": [0.573922872543335, 0.6326964497566223, 0.6536688804626465], "cos_apical_negg": [0.573922872543335, 0.6326964497566223, 0.6536688804626465], "cos_Ac_negg": [0.573922872543335, 0.6326964497566223, 0.6536688804626465], "r_norm": [0.16681884229183197, 0.16693252325057983, 0.1697375625371933]}, {"step": 6200, "epoch": 15, "train_loss": 0.0005040180403739214, "cos_r_negg": [0.5730463266372681, 0.6337961554527283, 0.6553614139556885], "cos_innovation_negg": [0.5730463266372681, 0.6337961554527283, 0.6553614139556885], "cos_apical_negg": [0.5730463266372681, 0.6337961554527283, 0.6553614139556885], "cos_Ac_negg": [0.5730463266372681, 0.6337961554527283, 0.6553614139556885], "r_norm": [0.16911694407463074, 0.16918209195137024, 0.17184799909591675]}, {"step": 6250, "epoch": 15, "train_loss": 0.00025355545221827924, "cos_r_negg": [0.5723828077316284, 0.6338577270507812, 0.6551005244255066], "cos_innovation_negg": [0.5723828077316284, 0.6338577270507812, 0.6551005244255066], "cos_apical_negg": [0.5723828077316284, 0.6338577270507812, 0.6551005244255066], "cos_Ac_negg": [0.5723828077316284, 0.6338577270507812, 0.6551005244255066], "r_norm": [0.16837534308433533, 0.1684740036725998, 0.17099611461162567]}, {"epoch_end": 15, "step": 6256, "test_acc": 0.973, "test_loss": 0.152428976726532}, {"step": 6300, "epoch": 16, "train_loss": 0.00046462833415716887, "cos_r_negg": [0.5710126757621765, 0.6337977051734924, 0.6550008058547974], "cos_innovation_negg": [0.5710126757621765, 0.6337977051734924, 0.6550008058547974], "cos_apical_negg": [0.5710126757621765, 0.6337977051734924, 0.6550008058547974], "cos_Ac_negg": [0.5710126757621765, 0.6337977051734924, 0.6550008058547974], "r_norm": [0.17040103673934937, 0.17053532600402832, 0.17295685410499573]}, {"step": 6350, "epoch": 16, "train_loss": 0.000234415361774154, "cos_r_negg": [0.5709201097488403, 0.6322741508483887, 0.6538553833961487], "cos_innovation_negg": [0.5709201097488403, 0.6322741508483887, 0.6538553833961487], "cos_apical_negg": [0.5709201097488403, 0.6322741508483887, 0.6538553833961487], "cos_Ac_negg": [0.5709201097488403, 0.6322741508483887, 0.6538553833961487], "r_norm": [0.1673671454191208, 0.16746604442596436, 0.1699988692998886]}, {"step": 6400, "epoch": 16, "train_loss": 0.000626576947979629, "cos_r_negg": [0.571091890335083, 0.6319248676300049, 0.6544192433357239], "cos_innovation_negg": [0.571091890335083, 0.6319248676300049, 0.6544192433357239], "cos_apical_negg": [0.571091890335083, 0.6319248676300049, 0.6544192433357239], "cos_Ac_negg": [0.571091890335083, 0.6319248676300049, 0.6544192433357239], "r_norm": [0.16541078686714172, 0.1655915230512619, 0.16831693053245544]}, {"step": 6450, "epoch": 16, "train_loss": 0.00015456035907845944, "cos_r_negg": [0.5720399618148804, 0.6331284046173096, 0.6552959680557251], "cos_innovation_negg": [0.5720399618148804, 0.6331284046173096, 0.6552959680557251], "cos_apical_negg": [0.5720399618148804, 0.6331284046173096, 0.6552959680557251], "cos_Ac_negg": [0.5720399618148804, 0.6331284046173096, 0.6552959680557251], "r_norm": [0.16582249104976654, 0.165988028049469, 0.16870304942131042]}, {"step": 6500, "epoch": 16, "train_loss": 0.0007416277658194304, "cos_r_negg": [0.5719582438468933, 0.6320109963417053, 0.6550554633140564], "cos_innovation_negg": [0.5719582438468933, 0.6320109963417053, 0.6550554633140564], "cos_apical_negg": [0.5719582438468933, 0.6320109963417053, 0.6550554633140564], "cos_Ac_negg": [0.5719582438468933, 0.6320109963417053, 0.6550554633140564], "r_norm": [0.16280189156532288, 0.16302241384983063, 0.1659965217113495]}, {"step": 6550, "epoch": 16, "train_loss": 0.000376436481019482, "cos_r_negg": [0.5721213817596436, 0.6297732591629028, 0.6530128717422485], "cos_innovation_negg": [0.5721213817596436, 0.6297732591629028, 0.6530128717422485], "cos_apical_negg": [0.5721213817596436, 0.6297732591629028, 0.6530128717422485], "cos_Ac_negg": [0.5721213817596436, 0.6297732591629028, 0.6530128717422485], "r_norm": [0.16683754324913025, 0.16688138246536255, 0.16975755989551544]}, {"step": 6600, "epoch": 16, "train_loss": 0.00029234637622721493, "cos_r_negg": [0.5725017189979553, 0.6305321455001831, 0.6528756618499756], "cos_innovation_negg": [0.5725017189979553, 0.6305321455001831, 0.6528756618499756], "cos_apical_negg": [0.5725017189979553, 0.6305321455001831, 0.6528756618499756], "cos_Ac_negg": [0.5725017189979553, 0.6305321455001831, 0.6528756618499756], "r_norm": [0.16798283159732819, 0.16818666458129883, 0.17092640697956085]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9725, "test_loss": 0.1537531560897827}, {"step": 6650, "epoch": 17, "train_loss": 0.00035680815926752985, "cos_r_negg": [0.5719821453094482, 0.6312331557273865, 0.6533434391021729], "cos_innovation_negg": [0.5719821453094482, 0.6312331557273865, 0.6533434391021729], "cos_apical_negg": [0.5719821453094482, 0.6312331557273865, 0.6533434391021729], "cos_Ac_negg": [0.5719821453094482, 0.6312331557273865, 0.6533434391021729], "r_norm": [0.16878892481327057, 0.1690538078546524, 0.17185090482234955]}, {"step": 6700, "epoch": 17, "train_loss": 0.00014642442693002522, "cos_r_negg": [0.5731503963470459, 0.63126540184021, 0.6534918546676636], "cos_innovation_negg": [0.5731503963470459, 0.63126540184021, 0.6534918546676636], "cos_apical_negg": [0.5731503963470459, 0.63126540184021, 0.6534918546676636], "cos_Ac_negg": [0.5731503963470459, 0.63126540184021, 0.6534918546676636], "r_norm": [0.17153877019882202, 0.17168214917182922, 0.1743067353963852]}, {"step": 6750, "epoch": 17, "train_loss": 0.0006596780149266124, "cos_r_negg": [0.5729934573173523, 0.6323469877243042, 0.6551302075386047], "cos_innovation_negg": [0.5729934573173523, 0.6323469877243042, 0.6551302075386047], "cos_apical_negg": [0.5729934573173523, 0.6323469877243042, 0.6551302075386047], "cos_Ac_negg": [0.5729934573173523, 0.6323469877243042, 0.6551302075386047], "r_norm": [0.16913259029388428, 0.16921156644821167, 0.1718774139881134]}, {"step": 6800, "epoch": 17, "train_loss": 0.0004942967207171023, "cos_r_negg": [0.5716614127159119, 0.6313984990119934, 0.6547229290008545], "cos_innovation_negg": [0.5716614127159119, 0.6313984990119934, 0.6547229290008545], "cos_apical_negg": [0.5716614127159119, 0.6313984990119934, 0.6547229290008545], "cos_Ac_negg": [0.5716614127159119, 0.6313984990119934, 0.6547229290008545], "r_norm": [0.16718757152557373, 0.16737492382526398, 0.17025628685951233]}, {"step": 6850, "epoch": 17, "train_loss": 0.00023993398644961417, "cos_r_negg": [0.5728077292442322, 0.6303594708442688, 0.6544203758239746], "cos_innovation_negg": [0.5728077292442322, 0.6303594708442688, 0.6544203758239746], "cos_apical_negg": [0.5728077292442322, 0.6303594708442688, 0.6544203758239746], "cos_Ac_negg": [0.5728077292442322, 0.6303594708442688, 0.6544203758239746], "r_norm": [0.16600564122200012, 0.16628627479076385, 0.16919074952602386]}, {"step": 6900, "epoch": 17, "train_loss": 0.0008615278638899326, "cos_r_negg": [0.5724399089813232, 0.6310233473777771, 0.6543644666671753], "cos_innovation_negg": [0.5724399089813232, 0.6310233473777771, 0.6543644666671753], "cos_apical_negg": [0.5724399089813232, 0.6310233473777771, 0.6543644666671753], "cos_Ac_negg": [0.5724399089813232, 0.6310233473777771, 0.6543644666671753], "r_norm": [0.16834154725074768, 0.16844511032104492, 0.17125029861927032]}, {"step": 6950, "epoch": 17, "train_loss": 0.0006292093312367797, "cos_r_negg": [0.573560893535614, 0.6310700178146362, 0.6544480323791504], "cos_innovation_negg": [0.573560893535614, 0.6310700178146362, 0.6544480323791504], "cos_apical_negg": [0.573560893535614, 0.6310700178146362, 0.6544480323791504], "cos_Ac_negg": [0.573560893535614, 0.6310700178146362, 0.6544480323791504], "r_norm": [0.16665855050086975, 0.16692867875099182, 0.1698058843612671]}, {"step": 7000, "epoch": 17, "train_loss": 0.00035835744347423315, "cos_r_negg": [0.5735301971435547, 0.6315364837646484, 0.6546908020973206], "cos_innovation_negg": [0.5735301971435547, 0.6315364837646484, 0.6546908020973206], "cos_apical_negg": [0.5735301971435547, 0.6315364837646484, 0.6546908020973206], "cos_Ac_negg": [0.5735301971435547, 0.6315364837646484, 0.6546908020973206], "r_norm": [0.16852936148643494, 0.1689625084400177, 0.17164099216461182]}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9723, "test_loss": 0.15637185096740722}, {"step": 7050, "epoch": 18, "train_loss": 0.00012607654207386076, "cos_r_negg": [0.5731619596481323, 0.6313391923904419, 0.6550973057746887], "cos_innovation_negg": [0.5731619596481323, 0.6313391923904419, 0.6550973057746887], "cos_apical_negg": [0.5731619596481323, 0.6313391923904419, 0.6550973057746887], "cos_Ac_negg": [0.5731619596481323, 0.6313391923904419, 0.6550973057746887], "r_norm": [0.16838355362415314, 0.16859287023544312, 0.1712210774421692]}, {"step": 7100, "epoch": 18, "train_loss": 0.00019493719446472824, "cos_r_negg": [0.5742883086204529, 0.6319400072097778, 0.6553126573562622], "cos_innovation_negg": [0.5742883086204529, 0.6319400072097778, 0.6553126573562622], "cos_apical_negg": [0.5742883086204529, 0.6319400072097778, 0.6553126573562622], "cos_Ac_negg": [0.5742883086204529, 0.6319400072097778, 0.6553126573562622], "r_norm": [0.16948509216308594, 0.1696128249168396, 0.1722295731306076]}, {"step": 7150, "epoch": 18, "train_loss": 0.000163051241543144, "cos_r_negg": [0.574087381362915, 0.6317781805992126, 0.6552608013153076], "cos_innovation_negg": [0.574087381362915, 0.6317781805992126, 0.6552608013153076], "cos_apical_negg": [0.574087381362915, 0.6317781805992126, 0.6552608013153076], "cos_Ac_negg": [0.574087381362915, 0.6317781805992126, 0.6552608013153076], "r_norm": [0.16941821575164795, 0.16951481997966766, 0.17218226194381714]}, {"step": 7200, "epoch": 18, "train_loss": 0.0001924895477714017, "cos_r_negg": [0.5750548839569092, 0.6325428485870361, 0.6561834812164307], "cos_innovation_negg": [0.5750548839569092, 0.6325428485870361, 0.6561834812164307], "cos_apical_negg": [0.5750548839569092, 0.6325428485870361, 0.6561834812164307], "cos_Ac_negg": [0.5750548839569092, 0.6325428485870361, 0.6561834812164307], "r_norm": [0.16810914874076843, 0.16815239191055298, 0.17091159522533417]}, {"step": 7250, "epoch": 18, "train_loss": 0.00044795096619054675, "cos_r_negg": [0.5753976106643677, 0.632246732711792, 0.6560173034667969], "cos_innovation_negg": [0.5753976106643677, 0.632246732711792, 0.6560173034667969], "cos_apical_negg": [0.5753976106643677, 0.632246732711792, 0.6560173034667969], "cos_Ac_negg": [0.5753976106643677, 0.632246732711792, 0.6560173034667969], "r_norm": [0.16924738883972168, 0.1692965030670166, 0.17200815677642822]}, {"step": 7300, "epoch": 18, "train_loss": 0.00033256024471484125, "cos_r_negg": [0.5756068229675293, 0.632030725479126, 0.6564499139785767], "cos_innovation_negg": [0.5756068229675293, 0.632030725479126, 0.6564499139785767], "cos_apical_negg": [0.5756068229675293, 0.632030725479126, 0.6564499139785767], "cos_Ac_negg": [0.5756068229675293, 0.632030725479126, 0.6564499139785767], "r_norm": [0.16817918419837952, 0.168365478515625, 0.171077698469162]}, {"step": 7350, "epoch": 18, "train_loss": 0.001629785867407918, "cos_r_negg": [0.5753393173217773, 0.631667971611023, 0.6558020710945129], "cos_innovation_negg": [0.5753393173217773, 0.631667971611023, 0.6558020710945129], "cos_apical_negg": [0.5753393173217773, 0.631667971611023, 0.6558020710945129], "cos_Ac_negg": [0.5753393173217773, 0.631667971611023, 0.6558020710945129], "r_norm": [0.1677895188331604, 0.16811978816986084, 0.17078498005867004]}, {"step": 7400, "epoch": 18, "train_loss": 0.0002279902109876275, "cos_r_negg": [0.5737911462783813, 0.6313401460647583, 0.6559534668922424], "cos_innovation_negg": [0.5737911462783813, 0.6313401460647583, 0.6559534668922424], "cos_apical_negg": [0.5737911462783813, 0.6313401460647583, 0.6559534668922424], "cos_Ac_negg": [0.5737911462783813, 0.6313401460647583, 0.6559534668922424], "r_norm": [0.16648533940315247, 0.1667291522026062, 0.169597789645195]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9726, "test_loss": 0.15768123512268067}, {"step": 7450, "epoch": 19, "train_loss": 0.0002906442095991224, "cos_r_negg": [0.5740014314651489, 0.6314606666564941, 0.6557754278182983], "cos_innovation_negg": [0.5740014314651489, 0.6314606666564941, 0.6557754278182983], "cos_apical_negg": [0.5740014314651489, 0.6314606666564941, 0.6557754278182983], "cos_Ac_negg": [0.5740014314651489, 0.6314606666564941, 0.6557754278182983], "r_norm": [0.1685255467891693, 0.16887801885604858, 0.17158332467079163]}, {"step": 7500, "epoch": 19, "train_loss": 0.00023658618738409132, "cos_r_negg": [0.5731000900268555, 0.6309593319892883, 0.6558160185813904], "cos_innovation_negg": [0.5731000900268555, 0.6309593319892883, 0.6558160185813904], "cos_apical_negg": [0.5731000900268555, 0.6309593319892883, 0.6558160185813904], "cos_Ac_negg": [0.5731000900268555, 0.6309593319892883, 0.6558160185813904], "r_norm": [0.16829325258731842, 0.1685289740562439, 0.17120498418807983]}, {"step": 7550, "epoch": 19, "train_loss": 0.00032023226958699524, "cos_r_negg": [0.5727131366729736, 0.6312599182128906, 0.6562078595161438], "cos_innovation_negg": [0.5727131366729736, 0.6312599182128906, 0.6562078595161438], "cos_apical_negg": [0.5727131366729736, 0.6312599182128906, 0.6562078595161438], "cos_Ac_negg": [0.5727131366729736, 0.6312599182128906, 0.6562078595161438], "r_norm": [0.16825324296951294, 0.1685229390859604, 0.171293705701828]}, {"step": 7600, "epoch": 19, "train_loss": 0.0001992911275010556, "cos_r_negg": [0.5745987892150879, 0.6312490701675415, 0.6561055183410645], "cos_innovation_negg": [0.5745987892150879, 0.6312490701675415, 0.6561055183410645], "cos_apical_negg": [0.5745987892150879, 0.6312490701675415, 0.6561055183410645], "cos_Ac_negg": [0.5745987892150879, 0.6312490701675415, 0.6561055183410645], "r_norm": [0.16642551124095917, 0.16673173010349274, 0.1696493923664093]}, {"step": 7650, "epoch": 19, "train_loss": 0.00038266839692369103, "cos_r_negg": [0.572625458240509, 0.6308154463768005, 0.6562432050704956], "cos_innovation_negg": [0.572625458240509, 0.6308154463768005, 0.6562432050704956], "cos_apical_negg": [0.572625458240509, 0.6308154463768005, 0.6562432050704956], "cos_Ac_negg": [0.572625458240509, 0.6308154463768005, 0.6562432050704956], "r_norm": [0.16767944395542145, 0.1678803265094757, 0.17070895433425903]}, {"step": 7700, "epoch": 19, "train_loss": 0.0005820182850584388, "cos_r_negg": [0.5731847286224365, 0.6314961910247803, 0.6563503742218018], "cos_innovation_negg": [0.5731847286224365, 0.6314961910247803, 0.6563503742218018], "cos_apical_negg": [0.5731847286224365, 0.6314961910247803, 0.6563503742218018], "cos_Ac_negg": [0.5731847286224365, 0.6314961910247803, 0.6563503742218018], "r_norm": [0.16659699380397797, 0.1667129546403885, 0.1695077121257782]}, {"step": 7750, "epoch": 19, "train_loss": 0.0005406605778262019, "cos_r_negg": [0.5728203058242798, 0.6310551762580872, 0.6561238765716553], "cos_innovation_negg": [0.5728203058242798, 0.6310551762580872, 0.6561238765716553], "cos_apical_negg": [0.5728203058242798, 0.6310551762580872, 0.6561238765716553], "cos_Ac_negg": [0.5728203058242798, 0.6310551762580872, 0.6561238765716553], "r_norm": [0.16873767971992493, 0.16883526742458344, 0.17158222198486328]}, {"step": 7800, "epoch": 19, "train_loss": 0.0003832356305792928, "cos_r_negg": [0.572131335735321, 0.6311428546905518, 0.6566826105117798], "cos_innovation_negg": [0.572131335735321, 0.6311428546905518, 0.6566826105117798], "cos_apical_negg": [0.572131335735321, 0.6311428546905518, 0.6566826105117798], "cos_Ac_negg": [0.572131335735321, 0.6311428546905518, 0.6566826105117798], "r_norm": [0.170294851064682, 0.17039145529270172, 0.17303243279457092]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9724, "test_loss": 0.15997611541748047}, {"step": 7850, "epoch": 20, "train_loss": 0.00026491557946428657, "cos_r_negg": [0.5744224786758423, 0.6311075091362, 0.6566098928451538], "cos_innovation_negg": [0.5744224786758423, 0.6311075091362, 0.6566098928451538], "cos_apical_negg": [0.5744224786758423, 0.6311075091362, 0.6566098928451538], "cos_Ac_negg": [0.5744224786758423, 0.6311075091362, 0.6566098928451538], "r_norm": [0.17049133777618408, 0.17072317004203796, 0.1733398586511612]}, {"step": 7900, "epoch": 20, "train_loss": 0.0002705724909901619, "cos_r_negg": [0.5738329291343689, 0.630517840385437, 0.6567674875259399], "cos_innovation_negg": [0.5738329291343689, 0.630517840385437, 0.6567674875259399], "cos_apical_negg": [0.5738329291343689, 0.630517840385437, 0.6567674875259399], "cos_Ac_negg": [0.5738329291343689, 0.630517840385437, 0.6567674875259399], "r_norm": [0.16907504200935364, 0.1693146973848343, 0.17207084596157074]}, {"step": 7950, "epoch": 20, "train_loss": 0.00014542050485033542, "cos_r_negg": [0.5745958089828491, 0.6311944723129272, 0.6565056443214417], "cos_innovation_negg": [0.5745958089828491, 0.6311944723129272, 0.6565056443214417], "cos_apical_negg": [0.5745958089828491, 0.6311944723129272, 0.6565056443214417], "cos_Ac_negg": [0.5745958089828491, 0.6311944723129272, 0.6565056443214417], "r_norm": [0.16791686415672302, 0.16819030046463013, 0.17101509869098663]}, {"step": 8000, "epoch": 20, "train_loss": 0.00014790311979595572, "cos_r_negg": [0.5738018751144409, 0.6312724351882935, 0.656771183013916], "cos_innovation_negg": [0.5738018751144409, 0.6312724351882935, 0.656771183013916], "cos_apical_negg": [0.5738018751144409, 0.6312724351882935, 0.656771183013916], "cos_Ac_negg": [0.5738018751144409, 0.6312724351882935, 0.656771183013916], "r_norm": [0.16704678535461426, 0.16737832129001617, 0.17023590207099915]}, {"step": 8050, "epoch": 20, "train_loss": 0.00039090419886633754, "cos_r_negg": [0.5744500160217285, 0.6316293478012085, 0.6571438312530518], "cos_innovation_negg": [0.5744500160217285, 0.6316293478012085, 0.6571438312530518], "cos_apical_negg": [0.5744500160217285, 0.6316293478012085, 0.6571438312530518], "cos_Ac_negg": [0.5744500160217285, 0.6316293478012085, 0.6571438312530518], "r_norm": [0.1719898134469986, 0.17224732041358948, 0.17480701208114624]}, {"step": 8100, "epoch": 20, "train_loss": 0.00018617307068780065, "cos_r_negg": [0.5738469362258911, 0.6321585178375244, 0.6570966243743896], "cos_innovation_negg": [0.5738469362258911, 0.6321585178375244, 0.6570966243743896], "cos_apical_negg": [0.5738469362258911, 0.6321585178375244, 0.6570966243743896], "cos_Ac_negg": [0.5738469362258911, 0.6321585178375244, 0.6570966243743896], "r_norm": [0.1716732382774353, 0.17180171608924866, 0.17438441514968872]}, {"step": 8150, "epoch": 20, "train_loss": 0.00018842022109311074, "cos_r_negg": [0.573945164680481, 0.6309248208999634, 0.6576639413833618], "cos_innovation_negg": [0.573945164680481, 0.6309248208999634, 0.6576639413833618], "cos_apical_negg": [0.573945164680481, 0.6309248208999634, 0.6576639413833618], "cos_Ac_negg": [0.573945164680481, 0.6309248208999634, 0.6576639413833618], "r_norm": [0.17069730162620544, 0.1708734929561615, 0.1734597086906433]}, {"step": 8200, "epoch": 20, "train_loss": 0.00033484186860732734, "cos_r_negg": [0.5730733871459961, 0.6307982206344604, 0.6577816009521484], "cos_innovation_negg": [0.5730733871459961, 0.6307982206344604, 0.6577816009521484], "cos_apical_negg": [0.5730733871459961, 0.6307982206344604, 0.6577816009521484], "cos_Ac_negg": [0.5730733871459961, 0.6307982206344604, 0.6577816009521484], "r_norm": [0.17185242474079132, 0.17186982929706573, 0.1744643747806549]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9726, "test_loss": 0.16079311389923095}, {"step": 8250, "epoch": 21, "train_loss": 0.0005908651510253549, "cos_r_negg": [0.5722845792770386, 0.6297252178192139, 0.6573386192321777], "cos_innovation_negg": [0.5722845792770386, 0.6297252178192139, 0.6573386192321777], "cos_apical_negg": [0.5722845792770386, 0.6297252178192139, 0.6573386192321777], "cos_Ac_negg": [0.5722845792770386, 0.6297252178192139, 0.6573386192321777], "r_norm": [0.17404545843601227, 0.17406626045703888, 0.17659235000610352]}, {"step": 8300, "epoch": 21, "train_loss": 0.00022507963876705617, "cos_r_negg": [0.573434054851532, 0.6306283473968506, 0.6577632427215576], "cos_innovation_negg": [0.573434054851532, 0.6306283473968506, 0.6577632427215576], "cos_apical_negg": [0.573434054851532, 0.6306283473968506, 0.6577632427215576], "cos_Ac_negg": [0.573434054851532, 0.6306283473968506, 0.6577632427215576], "r_norm": [0.17193426191806793, 0.17216390371322632, 0.17477238178253174]}, {"step": 8350, "epoch": 21, "train_loss": 0.0002939407713711262, "cos_r_negg": [0.5736374258995056, 0.6305879950523376, 0.6580498218536377], "cos_innovation_negg": [0.5736374258995056, 0.6305879950523376, 0.6580498218536377], "cos_apical_negg": [0.5736374258995056, 0.6305879950523376, 0.6580498218536377], "cos_Ac_negg": [0.5736374258995056, 0.6305879950523376, 0.6580498218536377], "r_norm": [0.1728018969297409, 0.1729303002357483, 0.1755070984363556]}, {"step": 8400, "epoch": 21, "train_loss": 0.0002807753626257181, "cos_r_negg": [0.5736789703369141, 0.6307692527770996, 0.6583400368690491], "cos_innovation_negg": [0.5736789703369141, 0.6307692527770996, 0.6583400368690491], "cos_apical_negg": [0.5736789703369141, 0.6307692527770996, 0.6583400368690491], "cos_Ac_negg": [0.5736789703369141, 0.6307692527770996, 0.6583400368690491], "r_norm": [0.17202100157737732, 0.1720433235168457, 0.17461639642715454]}, {"step": 8450, "epoch": 21, "train_loss": 0.00023880458320491016, "cos_r_negg": [0.5730299949645996, 0.630973219871521, 0.6586096286773682], "cos_innovation_negg": [0.5730299949645996, 0.630973219871521, 0.6586096286773682], "cos_apical_negg": [0.5730299949645996, 0.630973219871521, 0.6586096286773682], "cos_Ac_negg": [0.5730299949645996, 0.630973219871521, 0.6586096286773682], "r_norm": [0.16960856318473816, 0.16973790526390076, 0.17241095006465912]}, {"step": 8500, "epoch": 21, "train_loss": 0.0005070358165539801, "cos_r_negg": [0.5729165077209473, 0.6317052841186523, 0.6587579250335693], "cos_innovation_negg": [0.5729165077209473, 0.6317052841186523, 0.6587579250335693], "cos_apical_negg": [0.5729165077209473, 0.6317052841186523, 0.6587579250335693], "cos_Ac_negg": [0.5729165077209473, 0.6317052841186523, 0.6587579250335693], "r_norm": [0.17023533582687378, 0.17036719620227814, 0.17302711308002472]}, {"step": 8550, "epoch": 21, "train_loss": 0.0001448707189410925, "cos_r_negg": [0.5728161334991455, 0.6319823265075684, 0.6590566039085388], "cos_innovation_negg": [0.5728161334991455, 0.6319823265075684, 0.6590566039085388], "cos_apical_negg": [0.5728161334991455, 0.6319823265075684, 0.6590566039085388], "cos_Ac_negg": [0.5728161334991455, 0.6319823265075684, 0.6590566039085388], "r_norm": [0.1704685389995575, 0.17048707604408264, 0.17317277193069458]}, {"step": 8600, "epoch": 21, "train_loss": 0.0010817312868312001, "cos_r_negg": [0.5737299919128418, 0.6314330697059631, 0.659121036529541], "cos_innovation_negg": [0.5737299919128418, 0.6314330697059631, 0.659121036529541], "cos_apical_negg": [0.5737299919128418, 0.6314330697059631, 0.659121036529541], "cos_Ac_negg": [0.5737299919128418, 0.6314330697059631, 0.659121036529541], "r_norm": [0.16895702481269836, 0.1691034734249115, 0.1718699336051941]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9724, "test_loss": 0.16270455923080443}, {"step": 8650, "epoch": 22, "train_loss": 0.0001675389357842505, "cos_r_negg": [0.5730302333831787, 0.63084876537323, 0.6587589979171753], "cos_innovation_negg": [0.5730302333831787, 0.63084876537323, 0.6587589979171753], "cos_apical_negg": [0.5730302333831787, 0.63084876537323, 0.6587589979171753], "cos_Ac_negg": [0.5730302333831787, 0.63084876537323, 0.6587589979171753], "r_norm": [0.1706097573041916, 0.17072007060050964, 0.17341144382953644]}, {"step": 8700, "epoch": 22, "train_loss": 0.0002082207502098754, "cos_r_negg": [0.5731883645057678, 0.6308109760284424, 0.6585292816162109], "cos_innovation_negg": [0.5731883645057678, 0.6308109760284424, 0.6585292816162109], "cos_apical_negg": [0.5731883645057678, 0.6308109760284424, 0.6585292816162109], "cos_Ac_negg": [0.5731883645057678, 0.6308109760284424, 0.6585292816162109], "r_norm": [0.169408917427063, 0.16956083476543427, 0.17232757806777954]}, {"step": 8750, "epoch": 22, "train_loss": 0.00015429704217240214, "cos_r_negg": [0.5733445286750793, 0.631112813949585, 0.6590025424957275], "cos_innovation_negg": [0.5733445286750793, 0.631112813949585, 0.6590025424957275], "cos_apical_negg": [0.5733445286750793, 0.631112813949585, 0.6590025424957275], "cos_Ac_negg": [0.5733445286750793, 0.631112813949585, 0.6590025424957275], "r_norm": [0.16866351664066315, 0.16882938146591187, 0.1715840995311737]}, {"step": 8800, "epoch": 22, "train_loss": 0.00033808103762567043, "cos_r_negg": [0.5731734037399292, 0.6312898397445679, 0.6589257717132568], "cos_innovation_negg": [0.5731734037399292, 0.6312898397445679, 0.6589257717132568], "cos_apical_negg": [0.5731734037399292, 0.6312898397445679, 0.6589257717132568], "cos_Ac_negg": [0.5731734037399292, 0.6312898397445679, 0.6589257717132568], "r_norm": [0.16893541812896729, 0.1691654622554779, 0.17194125056266785]}, {"step": 8850, "epoch": 22, "train_loss": 0.00025564676616340876, "cos_r_negg": [0.5729660987854004, 0.6314924359321594, 0.6595416069030762], "cos_innovation_negg": [0.5729660987854004, 0.6314924359321594, 0.6595416069030762], "cos_apical_negg": [0.5729660987854004, 0.6314924359321594, 0.6595416069030762], "cos_Ac_negg": [0.5729660987854004, 0.6314924359321594, 0.6595416069030762], "r_norm": [0.16702939569950104, 0.1672196388244629, 0.1701136827468872]}, {"step": 8900, "epoch": 22, "train_loss": 0.0006284746341407299, "cos_r_negg": [0.5724520683288574, 0.63105309009552, 0.6595825552940369], "cos_innovation_negg": [0.5724520683288574, 0.63105309009552, 0.6595825552940369], "cos_apical_negg": [0.5724520683288574, 0.63105309009552, 0.6595825552940369], "cos_Ac_negg": [0.5724520683288574, 0.63105309009552, 0.6595825552940369], "r_norm": [0.16906385123729706, 0.16919606924057007, 0.17197179794311523]}, {"step": 8950, "epoch": 22, "train_loss": 0.0003338629030622542, "cos_r_negg": [0.5731325149536133, 0.6304562091827393, 0.6594637632369995], "cos_innovation_negg": [0.5731325149536133, 0.6304562091827393, 0.6594637632369995], "cos_apical_negg": [0.5731325149536133, 0.6304562091827393, 0.6594637632369995], "cos_Ac_negg": [0.5731325149536133, 0.6304562091827393, 0.6594637632369995], "r_norm": [0.17018775641918182, 0.17026665806770325, 0.17296814918518066]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9726, "test_loss": 0.16367367877960204}, {"step": 9000, "epoch": 23, "train_loss": 0.00012306372809689492, "cos_r_negg": [0.5732861757278442, 0.6309744119644165, 0.6596060395240784], "cos_innovation_negg": [0.5732861757278442, 0.6309744119644165, 0.6596060395240784], "cos_apical_negg": [0.5732861757278442, 0.6309744119644165, 0.6596060395240784], "cos_Ac_negg": [0.5732861757278442, 0.6309744119644165, 0.6596060395240784], "r_norm": [0.17180520296096802, 0.17197619378566742, 0.1746497005224228]}, {"step": 9050, "epoch": 23, "train_loss": 0.00021543525508604944, "cos_r_negg": [0.5738906860351562, 0.6322429776191711, 0.6602301597595215], "cos_innovation_negg": [0.5738906860351562, 0.6322429776191711, 0.6602301597595215], "cos_apical_negg": [0.5738906860351562, 0.6322429776191711, 0.6602301597595215], "cos_Ac_negg": [0.5738906860351562, 0.6322429776191711, 0.6602301597595215], "r_norm": [0.17010778188705444, 0.17020820081233978, 0.17294099926948547]}, {"step": 9100, "epoch": 23, "train_loss": 0.00014658784493803978, "cos_r_negg": [0.5729867219924927, 0.6311306357383728, 0.660011351108551], "cos_innovation_negg": [0.5729867219924927, 0.6311306357383728, 0.660011351108551], "cos_apical_negg": [0.5729867219924927, 0.6311306357383728, 0.660011351108551], "cos_Ac_negg": [0.5729867219924927, 0.6311306357383728, 0.660011351108551], "r_norm": [0.1702665388584137, 0.17035187780857086, 0.17307552695274353]}, {"step": 9150, "epoch": 23, "train_loss": 0.00017059207311831415, "cos_r_negg": [0.5737914443016052, 0.6311681270599365, 0.659088134765625], "cos_innovation_negg": [0.5737914443016052, 0.6311681270599365, 0.659088134765625], "cos_apical_negg": [0.5737914443016052, 0.6311681270599365, 0.659088134765625], "cos_Ac_negg": [0.5737914443016052, 0.6311681270599365, 0.659088134765625], "r_norm": [0.17053326964378357, 0.1706605851650238, 0.1733914464712143]}, {"step": 9200, "epoch": 23, "train_loss": 0.0007497973274439573, "cos_r_negg": [0.5733703374862671, 0.6317344903945923, 0.6597766280174255], "cos_innovation_negg": [0.5733703374862671, 0.6317344903945923, 0.6597766280174255], "cos_apical_negg": [0.5733703374862671, 0.6317344903945923, 0.6597766280174255], "cos_Ac_negg": [0.5733703374862671, 0.6317344903945923, 0.6597766280174255], "r_norm": [0.16936790943145752, 0.16950680315494537, 0.17231068015098572]}, {"step": 9250, "epoch": 23, "train_loss": 0.00021510539227165282, "cos_r_negg": [0.5747102499008179, 0.6321823596954346, 0.660497784614563], "cos_innovation_negg": [0.5747102499008179, 0.6321823596954346, 0.660497784614563], "cos_apical_negg": [0.5747102499008179, 0.6321823596954346, 0.660497784614563], "cos_Ac_negg": [0.5747102499008179, 0.6321823596954346, 0.660497784614563], "r_norm": [0.16999442875385284, 0.17007119953632355, 0.17281374335289001]}, {"step": 9300, "epoch": 23, "train_loss": 0.00020478414080571383, "cos_r_negg": [0.5751608610153198, 0.6327560544013977, 0.6604565382003784], "cos_innovation_negg": [0.5751608610153198, 0.6327560544013977, 0.6604565382003784], "cos_apical_negg": [0.5751608610153198, 0.6327560544013977, 0.6604565382003784], "cos_Ac_negg": [0.5751608610153198, 0.6327560544013977, 0.6604565382003784], "r_norm": [0.17082920670509338, 0.17102152109146118, 0.17368584871292114]}, {"step": 9350, "epoch": 23, "train_loss": 0.00011449891462689266, "cos_r_negg": [0.5744506120681763, 0.6322755813598633, 0.6605048179626465], "cos_innovation_negg": [0.5744506120681763, 0.6322755813598633, 0.6605048179626465], "cos_apical_negg": [0.5744506120681763, 0.6322755813598633, 0.6605048179626465], "cos_Ac_negg": [0.5744506120681763, 0.6322755813598633, 0.6605048179626465], "r_norm": [0.1725519597530365, 0.17265503108501434, 0.17526201903820038]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9723, "test_loss": 0.16502937450408936}, {"step": 9400, "epoch": 24, "train_loss": 0.00020532113558147103, "cos_r_negg": [0.5746176838874817, 0.6315740346908569, 0.6602378487586975], "cos_innovation_negg": [0.5746176838874817, 0.6315740346908569, 0.6602378487586975], "cos_apical_negg": [0.5746176838874817, 0.6315740346908569, 0.6602378487586975], "cos_Ac_negg": [0.5746176838874817, 0.6315740346908569, 0.6602378487586975], "r_norm": [0.17178013920783997, 0.1719052791595459, 0.17462341487407684]}, {"step": 9450, "epoch": 24, "train_loss": 0.00014097437087912112, "cos_r_negg": [0.574904203414917, 0.6317377686500549, 0.6601996421813965], "cos_innovation_negg": [0.574904203414917, 0.6317377686500549, 0.6601996421813965], "cos_apical_negg": [0.574904203414917, 0.6317377686500549, 0.6601996421813965], "cos_Ac_negg": [0.574904203414917, 0.6317377686500549, 0.6601996421813965], "r_norm": [0.1719578206539154, 0.17218399047851562, 0.17489197850227356]}, {"step": 9500, "epoch": 24, "train_loss": 0.00012655019236262888, "cos_r_negg": [0.574950098991394, 0.6320195198059082, 0.6602826118469238], "cos_innovation_negg": [0.574950098991394, 0.6320195198059082, 0.6602826118469238], "cos_apical_negg": [0.574950098991394, 0.6320195198059082, 0.6602826118469238], "cos_Ac_negg": [0.574950098991394, 0.6320195198059082, 0.6602826118469238], "r_norm": [0.1718227118253708, 0.1719263643026352, 0.1746213585138321]}, {"step": 9550, "epoch": 24, "train_loss": 0.00029193732189014554, "cos_r_negg": [0.5750663876533508, 0.6324639320373535, 0.6611915826797485], "cos_innovation_negg": [0.5750663876533508, 0.6324639320373535, 0.6611915826797485], "cos_apical_negg": [0.5750663876533508, 0.6324639320373535, 0.6611915826797485], "cos_Ac_negg": [0.5750663876533508, 0.6324639320373535, 0.6611915826797485], "r_norm": [0.1709003448486328, 0.17100916802883148, 0.17373064160346985]}, {"step": 9600, "epoch": 24, "train_loss": 0.00019516726024448872, "cos_r_negg": [0.5746427774429321, 0.6321399807929993, 0.6604310274124146], "cos_innovation_negg": [0.5746427774429321, 0.6321399807929993, 0.6604310274124146], "cos_apical_negg": [0.5746427774429321, 0.6321399807929993, 0.6604310274124146], "cos_Ac_negg": [0.5746427774429321, 0.6321399807929993, 0.6604310274124146], "r_norm": [0.17114107310771942, 0.17132070660591125, 0.17399077117443085]}, {"step": 9650, "epoch": 24, "train_loss": 0.00021931526134721935, "cos_r_negg": [0.5751739740371704, 0.6325116753578186, 0.6612073183059692], "cos_innovation_negg": [0.5751739740371704, 0.6325116753578186, 0.6612073183059692], "cos_apical_negg": [0.5751739740371704, 0.6325116753578186, 0.6612073183059692], "cos_Ac_negg": [0.5751739740371704, 0.6325116753578186, 0.6612073183059692], "r_norm": [0.16939380764961243, 0.16971521079540253, 0.17245979607105255]}, {"step": 9700, "epoch": 24, "train_loss": 0.00014631915837526321, "cos_r_negg": [0.5747259855270386, 0.6322188377380371, 0.6603038311004639], "cos_innovation_negg": [0.5747259855270386, 0.6322188377380371, 0.6603038311004639], "cos_apical_negg": [0.5747259855270386, 0.6322188377380371, 0.6603038311004639], "cos_Ac_negg": [0.5747259855270386, 0.6322188377380371, 0.6603038311004639], "r_norm": [0.16992944478988647, 0.170170858502388, 0.17288674414157867]}, {"step": 9750, "epoch": 24, "train_loss": 0.0004719137796200812, "cos_r_negg": [0.5745378732681274, 0.631935715675354, 0.6608942747116089], "cos_innovation_negg": [0.5745378732681274, 0.631935715675354, 0.6608942747116089], "cos_apical_negg": [0.5745378732681274, 0.631935715675354, 0.6608942747116089], "cos_Ac_negg": [0.5745378732681274, 0.631935715675354, 0.6608942747116089], "r_norm": [0.17186102271080017, 0.17197175323963165, 0.17464441061019897]}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9722, "test_loss": 0.16600589752197265}], "final": {"test_acc": 0.9722, "test_loss": 0.16600589752197265, "wall_s": 17.116204738616943, "cos_r_negg": [0.5753055214881897, 0.6319008469581604, 0.6606847643852234], "cos_innovation_negg": [0.5753055214881897, 0.6319008469581604, 0.6606847643852234], "cos_apical_negg": [0.5753055214881897, 0.6319008469581604, 0.6606847643852234], "cos_Ac_negg": [0.5753055214881897, 0.6319008469581604, 0.6606847643852234]}} \ No newline at end of file diff --git a/results/ep_match_v1_mnist_sdil_w256_d3_s0.json b/results/ep_match_v1_mnist_sdil_w256_d3_s0.json new file mode 100644 index 0000000..410e15a --- /dev/null +++ b/results/ep_match_v1_mnist_sdil_w256_d3_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_match_v1_mnist_sdil_w256_d3_s0", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3990068435668945, "cos_r_negg": [-0.019123679026961327, -0.020858976989984512, 0.03404799848794937], "cos_innovation_negg": [-0.019123679026961327, -0.020858976989984512, 0.03404799848794937], "cos_apical_negg": [-0.019123679026961327, -0.020858976989984512, 0.03404799848794937], "cos_Ac_negg": [-0.019123679026961327, -0.020858976989984512, 0.03404799848794937], "r_norm": [4.906693935394287, 4.836459159851074, 4.82870626449585], "ldr": {"dL_sdil": -0.012000560760498047, "dL_bp": -0.24264240264892578, "ratio": 0.0494578055174528}}, {"step": 50, "epoch": 0, "train_loss": 0.557349681854248, "cos_r_negg": [0.4386722445487976, 0.35197168588638306, 0.3038873076438904], "cos_innovation_negg": [0.4386722445487976, 0.35197168588638306, 0.3038873076438904], "cos_apical_negg": [0.4386722445487976, 0.35197168588638306, 0.3038873076438904], "cos_Ac_negg": [0.4386722445487976, 0.35197168588638306, 0.3038873076438904], "r_norm": [1.1577706336975098, 1.1234707832336426, 1.1238982677459717]}, {"step": 100, "epoch": 0, "train_loss": 0.21420708298683167, "cos_r_negg": [0.497479647397995, 0.40160071849823, 0.33492982387542725], "cos_innovation_negg": [0.497479647397995, 0.40160071849823, 0.33492982387542725], "cos_apical_negg": [0.497479647397995, 0.40160071849823, 0.33492982387542725], "cos_Ac_negg": [0.497479647397995, 0.40160071849823, 0.33492982387542725], "r_norm": [0.9483741521835327, 0.9255150556564331, 0.9265471696853638]}, {"step": 150, "epoch": 0, "train_loss": 0.1765260249376297, "cos_r_negg": [0.4874778985977173, 0.4004327058792114, 0.35426419973373413], "cos_innovation_negg": [0.4874778985977173, 0.4004327058792114, 0.35426419973373413], "cos_apical_negg": [0.4874778985977173, 0.4004327058792114, 0.35426419973373413], "cos_Ac_negg": [0.4874778985977173, 0.4004327058792114, 0.35426419973373413], "r_norm": [0.7939279079437256, 0.7757101058959961, 0.7750244140625]}, {"step": 200, "epoch": 0, "train_loss": 0.30060887336730957, "cos_r_negg": [0.5012128353118896, 0.43122828006744385, 0.3807114362716675], "cos_innovation_negg": [0.5012128353118896, 0.43122828006744385, 0.3807114362716675], "cos_apical_negg": [0.5012128353118896, 0.43122828006744385, 0.3807114362716675], "cos_Ac_negg": [0.5012128353118896, 0.43122828006744385, 0.3807114362716675], "r_norm": [0.7343144416809082, 0.715130090713501, 0.716572642326355]}, {"step": 250, "epoch": 0, "train_loss": 0.375924289226532, "cos_r_negg": [0.515765905380249, 0.46645665168762207, 0.4039836525917053], "cos_innovation_negg": [0.515765905380249, 0.46645665168762207, 0.4039836525917053], "cos_apical_negg": [0.515765905380249, 0.46645665168762207, 0.4039836525917053], "cos_Ac_negg": [0.515765905380249, 0.46645665168762207, 0.4039836525917053], "r_norm": [0.6069438457489014, 0.5937045216560364, 0.593670666217804], "ldr": {"dL_sdil": -0.011112242937088013, "dL_bp": -0.008948862552642822, "ratio": 1.2417492023951457}}, {"step": 300, "epoch": 0, "train_loss": 0.20754894614219666, "cos_r_negg": [0.49835503101348877, 0.4570460319519043, 0.4064018130302429], "cos_innovation_negg": [0.49835503101348877, 0.4570460319519043, 0.4064018130302429], "cos_apical_negg": [0.49835503101348877, 0.4570460319519043, 0.4064018130302429], "cos_Ac_negg": [0.49835503101348877, 0.4570460319519043, 0.4064018130302429], "r_norm": [0.6175251603126526, 0.6042684316635132, 0.6032040119171143]}, {"step": 350, "epoch": 0, "train_loss": 0.2702471911907196, "cos_r_negg": [0.5160802602767944, 0.45728257298469543, 0.4080795645713806], "cos_innovation_negg": [0.5160802602767944, 0.45728257298469543, 0.4080795645713806], "cos_apical_negg": [0.5160802602767944, 0.45728257298469543, 0.4080795645713806], "cos_Ac_negg": [0.5160802602767944, 0.45728257298469543, 0.4080795645713806], "r_norm": [0.6086537837982178, 0.5989933609962463, 0.5962319374084473]}, {"epoch_end": 0, "step": 391, "test_acc": 0.9434, "test_loss": 0.19921441268920898}, {"step": 400, "epoch": 1, "train_loss": 0.14611996710300446, "cos_r_negg": [0.5034387707710266, 0.46879106760025024, 0.40439730882644653], "cos_innovation_negg": [0.5034387707710266, 0.46879106760025024, 0.40439730882644653], "cos_apical_negg": [0.5034387707710266, 0.46879106760025024, 0.40439730882644653], "cos_Ac_negg": [0.5034387707710266, 0.46879106760025024, 0.40439730882644653], "r_norm": [0.5827618837356567, 0.5738776326179504, 0.5706467628479004]}, {"step": 450, "epoch": 1, "train_loss": 0.1880314201116562, "cos_r_negg": [0.506068766117096, 0.47526615858078003, 0.4247303903102875], "cos_innovation_negg": [0.506068766117096, 0.47526615858078003, 0.4247303903102875], "cos_apical_negg": [0.506068766117096, 0.47526615858078003, 0.4247303903102875], "cos_Ac_negg": [0.506068766117096, 0.47526615858078003, 0.4247303903102875], "r_norm": [0.48847055435180664, 0.4775075614452362, 0.4765704274177551]}, {"step": 500, "epoch": 1, "train_loss": 0.1368517130613327, "cos_r_negg": [0.4988791048526764, 0.48932552337646484, 0.4386179447174072], "cos_innovation_negg": [0.4988791048526764, 0.48932552337646484, 0.4386179447174072], "cos_apical_negg": [0.4988791048526764, 0.48932552337646484, 0.4386179447174072], "cos_Ac_negg": [0.4988791048526764, 0.48932552337646484, 0.4386179447174072], "r_norm": [0.42691463232040405, 0.4182046353816986, 0.41749975085258484], "ldr": {"dL_sdil": -0.006287813186645508, "dL_bp": -0.005642101168632507, "ratio": 1.1144453101271674}}, {"step": 550, "epoch": 1, "train_loss": 0.11181455105543137, "cos_r_negg": [0.5051651000976562, 0.5064424276351929, 0.4540102183818817], "cos_innovation_negg": [0.5051651000976562, 0.5064424276351929, 0.4540102183818817], "cos_apical_negg": [0.5051651000976562, 0.5064424276351929, 0.4540102183818817], "cos_Ac_negg": [0.5051651000976562, 0.5064424276351929, 0.4540102183818817], "r_norm": [0.4640708267688751, 0.45029354095458984, 0.4507724642753601]}, {"step": 600, "epoch": 1, "train_loss": 0.1023593544960022, "cos_r_negg": [0.5421346426010132, 0.5176475048065186, 0.47495079040527344], "cos_innovation_negg": [0.5421346426010132, 0.5176475048065186, 0.47495079040527344], "cos_apical_negg": [0.5421346426010132, 0.5176475048065186, 0.47495079040527344], "cos_Ac_negg": [0.5421346426010132, 0.5176475048065186, 0.47495079040527344], "r_norm": [0.533541202545166, 0.518498420715332, 0.5211732387542725]}, {"step": 650, "epoch": 1, "train_loss": 0.11898520588874817, "cos_r_negg": [0.5031847953796387, 0.4982442259788513, 0.45316529273986816], "cos_innovation_negg": [0.5031847953796387, 0.4982442259788513, 0.45316529273986816], "cos_apical_negg": [0.5031847953796387, 0.4982442259788513, 0.45316529273986816], "cos_Ac_negg": [0.5031847953796387, 0.4982442259788513, 0.45316529273986816], "r_norm": [0.40893441438674927, 0.3998851776123047, 0.3993779718875885]}, {"step": 700, "epoch": 1, "train_loss": 0.18386134505271912, "cos_r_negg": [0.51450115442276, 0.5159072875976562, 0.4532162845134735], "cos_innovation_negg": [0.51450115442276, 0.5159072875976562, 0.4532162845134735], "cos_apical_negg": [0.51450115442276, 0.5159072875976562, 0.4532162845134735], "cos_Ac_negg": [0.51450115442276, 0.5159072875976562, 0.4532162845134735], "r_norm": [0.38692131638526917, 0.3787069022655487, 0.37830400466918945]}, {"step": 750, "epoch": 1, "train_loss": 0.16645602881908417, "cos_r_negg": [0.528334379196167, 0.5304737091064453, 0.4670729637145996], "cos_innovation_negg": [0.528334379196167, 0.5304737091064453, 0.4670729637145996], "cos_apical_negg": [0.528334379196167, 0.5304737091064453, 0.4670729637145996], "cos_Ac_negg": [0.528334379196167, 0.5304737091064453, 0.4670729637145996], "r_norm": [0.46389123797416687, 0.45168060064315796, 0.4517902135848999], "ldr": {"dL_sdil": -0.013064056634902954, "dL_bp": -0.011553913354873657, "ratio": 1.1307040509692148}}, {"epoch_end": 1, "step": 782, "test_acc": 0.9537, "test_loss": 0.15850417404174805}, {"step": 800, "epoch": 2, "train_loss": 0.10791650414466858, "cos_r_negg": [0.5138213634490967, 0.5242051482200623, 0.46214574575424194], "cos_innovation_negg": [0.5138213634490967, 0.5242051482200623, 0.46214574575424194], "cos_apical_negg": [0.5138213634490967, 0.5242051482200623, 0.46214574575424194], "cos_Ac_negg": [0.5138213634490967, 0.5242051482200623, 0.46214574575424194], "r_norm": [0.38799065351486206, 0.37843409180641174, 0.3786221146583557]}, {"step": 850, "epoch": 2, "train_loss": 0.09848670661449432, "cos_r_negg": [0.5332177877426147, 0.5368587970733643, 0.4839399755001068], "cos_innovation_negg": [0.5332177877426147, 0.5368587970733643, 0.4839399755001068], "cos_apical_negg": [0.5332177877426147, 0.5368587970733643, 0.4839399755001068], "cos_Ac_negg": [0.5332177877426147, 0.5368587970733643, 0.4839399755001068], "r_norm": [0.3388267457485199, 0.3327179253101349, 0.3324386477470398]}, {"step": 900, "epoch": 2, "train_loss": 0.058545906096696854, "cos_r_negg": [0.5411534309387207, 0.5528039336204529, 0.502867579460144], "cos_innovation_negg": [0.5411534309387207, 0.5528039336204529, 0.502867579460144], "cos_apical_negg": [0.5411534309387207, 0.5528039336204529, 0.502867579460144], "cos_Ac_negg": [0.5411534309387207, 0.5528039336204529, 0.502867579460144], "r_norm": [0.4123993515968323, 0.4040146470069885, 0.4006704092025757]}, {"step": 950, "epoch": 2, "train_loss": 0.11710601300001144, "cos_r_negg": [0.5351316928863525, 0.5573509931564331, 0.5020351409912109], "cos_innovation_negg": [0.5351316928863525, 0.5573509931564331, 0.5020351409912109], "cos_apical_negg": [0.5351316928863525, 0.5573509931564331, 0.5020351409912109], "cos_Ac_negg": [0.5351316928863525, 0.5573509931564331, 0.5020351409912109], "r_norm": [0.29837918281555176, 0.2912159562110901, 0.29181450605392456]}, {"step": 1000, "epoch": 2, "train_loss": 0.1661570519208908, "cos_r_negg": [0.5447400808334351, 0.5624825954437256, 0.5104426145553589], "cos_innovation_negg": [0.5447400808334351, 0.5624825954437256, 0.5104426145553589], "cos_apical_negg": [0.5447400808334351, 0.5624825954437256, 0.5104426145553589], "cos_Ac_negg": [0.5447400808334351, 0.5624825954437256, 0.5104426145553589], "r_norm": [0.3730558753013611, 0.36110544204711914, 0.3602294325828552], "ldr": {"dL_sdil": -0.009133651852607727, "dL_bp": -0.00839608907699585, "ratio": 1.0878459922051922}}, {"step": 1050, "epoch": 2, "train_loss": 0.09447814524173737, "cos_r_negg": [0.532267689704895, 0.5573719143867493, 0.5009351968765259], "cos_innovation_negg": [0.532267689704895, 0.5573719143867493, 0.5009351968765259], "cos_apical_negg": [0.532267689704895, 0.5573719143867493, 0.5009351968765259], "cos_Ac_negg": [0.532267689704895, 0.5573719143867493, 0.5009351968765259], "r_norm": [0.3251861035823822, 0.3169657588005066, 0.31489551067352295]}, {"step": 1100, "epoch": 2, "train_loss": 0.13861429691314697, "cos_r_negg": [0.5416281223297119, 0.5739995241165161, 0.5101684927940369], "cos_innovation_negg": [0.5416281223297119, 0.5739995241165161, 0.5101684927940369], "cos_apical_negg": [0.5416281223297119, 0.5739995241165161, 0.5101684927940369], "cos_Ac_negg": [0.5416281223297119, 0.5739995241165161, 0.5101684927940369], "r_norm": [0.29859596490859985, 0.29098761081695557, 0.29072943329811096]}, {"step": 1150, "epoch": 2, "train_loss": 0.11408502608537674, "cos_r_negg": [0.5508650541305542, 0.592372715473175, 0.5120702385902405], "cos_innovation_negg": [0.5508650541305542, 0.592372715473175, 0.5120702385902405], "cos_apical_negg": [0.5508650541305542, 0.592372715473175, 0.5120702385902405], "cos_Ac_negg": [0.5508650541305542, 0.592372715473175, 0.5120702385902405], "r_norm": [0.2907700538635254, 0.282972514629364, 0.28388410806655884]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9621, "test_loss": 0.12575177898406983}, {"step": 1200, "epoch": 3, "train_loss": 0.06491747498512268, "cos_r_negg": [0.543096661567688, 0.5910171270370483, 0.5228492617607117], "cos_innovation_negg": [0.543096661567688, 0.5910171270370483, 0.5228492617607117], "cos_apical_negg": [0.543096661567688, 0.5910171270370483, 0.5228492617607117], "cos_Ac_negg": [0.543096661567688, 0.5910171270370483, 0.5228492617607117], "r_norm": [0.33233359456062317, 0.3243071138858795, 0.3222692608833313]}, {"step": 1250, "epoch": 3, "train_loss": 0.10036350786685944, "cos_r_negg": [0.5536779165267944, 0.59885174036026, 0.5401077270507812], "cos_innovation_negg": [0.5536779165267944, 0.59885174036026, 0.5401077270507812], "cos_apical_negg": [0.5536779165267944, 0.59885174036026, 0.5401077270507812], "cos_Ac_negg": [0.5536779165267944, 0.59885174036026, 0.5401077270507812], "r_norm": [0.2831721603870392, 0.2748281955718994, 0.2751358151435852], "ldr": {"dL_sdil": -0.008049741387367249, "dL_bp": -0.00775706022977829, "ratio": 1.0377309378706892}}, {"step": 1300, "epoch": 3, "train_loss": 0.05750660598278046, "cos_r_negg": [0.5708739757537842, 0.5975054502487183, 0.5410779714584351], "cos_innovation_negg": [0.5708739757537842, 0.5975054502487183, 0.5410779714584351], "cos_apical_negg": [0.5708739757537842, 0.5975054502487183, 0.5410779714584351], "cos_Ac_negg": [0.5708739757537842, 0.5975054502487183, 0.5410779714584351], "r_norm": [0.25697198510169983, 0.24996653199195862, 0.24974967539310455]}, {"step": 1350, "epoch": 3, "train_loss": 0.05158136039972305, "cos_r_negg": [0.5574784874916077, 0.5967764854431152, 0.5432888865470886], "cos_innovation_negg": [0.5574784874916077, 0.5967764854431152, 0.5432888865470886], "cos_apical_negg": [0.5574784874916077, 0.5967764854431152, 0.5432888865470886], "cos_Ac_negg": [0.5574784874916077, 0.5967764854431152, 0.5432888865470886], "r_norm": [0.2602882385253906, 0.2532706558704376, 0.2542128562927246]}, {"step": 1400, "epoch": 3, "train_loss": 0.04589533433318138, "cos_r_negg": [0.555349588394165, 0.5991731882095337, 0.5436723232269287], "cos_innovation_negg": [0.555349588394165, 0.5991731882095337, 0.5436723232269287], "cos_apical_negg": [0.555349588394165, 0.5991731882095337, 0.5436723232269287], "cos_Ac_negg": [0.555349588394165, 0.5991731882095337, 0.5436723232269287], "r_norm": [0.25799885392189026, 0.2506404221057892, 0.2510306239128113]}, {"step": 1450, "epoch": 3, "train_loss": 0.040470190346241, "cos_r_negg": [0.5421226620674133, 0.5957026481628418, 0.5470224618911743], "cos_innovation_negg": [0.5421226620674133, 0.5957026481628418, 0.5470224618911743], "cos_apical_negg": [0.5421226620674133, 0.5957026481628418, 0.5470224618911743], "cos_Ac_negg": [0.5421226620674133, 0.5957026481628418, 0.5470224618911743], "r_norm": [0.27448707818984985, 0.2665136158466339, 0.26880836486816406]}, {"step": 1500, "epoch": 3, "train_loss": 0.06418357789516449, "cos_r_negg": [0.5400217771530151, 0.5873705148696899, 0.5390570759773254], "cos_innovation_negg": [0.5400217771530151, 0.5873705148696899, 0.5390570759773254], "cos_apical_negg": [0.5400217771530151, 0.5873705148696899, 0.5390570759773254], "cos_Ac_negg": [0.5400217771530151, 0.5873705148696899, 0.5390570759773254], "r_norm": [0.23810845613479614, 0.23006708920001984, 0.23080921173095703], "ldr": {"dL_sdil": -0.005829907953739166, "dL_bp": -0.005748338997364044, "ratio": 1.0141900045234853}}, {"step": 1550, "epoch": 3, "train_loss": 0.08492086827754974, "cos_r_negg": [0.5419299006462097, 0.5801759362220764, 0.5428967475891113], "cos_innovation_negg": [0.5419299006462097, 0.5801759362220764, 0.5428967475891113], "cos_apical_negg": [0.5419299006462097, 0.5801759362220764, 0.5428967475891113], "cos_Ac_negg": [0.5419299006462097, 0.5801759362220764, 0.5428967475891113], "r_norm": [0.26595139503479004, 0.25891241431236267, 0.25652170181274414]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9606, "test_loss": 0.13444870491027833}, {"step": 1600, "epoch": 4, "train_loss": 0.07047419995069504, "cos_r_negg": [0.5622004270553589, 0.5888009071350098, 0.5449429750442505], "cos_innovation_negg": [0.5622004270553589, 0.5888009071350098, 0.5449429750442505], "cos_apical_negg": [0.5622004270553589, 0.5888009071350098, 0.5449429750442505], "cos_Ac_negg": [0.5622004270553589, 0.5888009071350098, 0.5449429750442505], "r_norm": [0.2528415322303772, 0.2463521510362625, 0.2465616911649704]}, {"step": 1650, "epoch": 4, "train_loss": 0.07882709056138992, "cos_r_negg": [0.5616379380226135, 0.5921094417572021, 0.562293291091919], "cos_innovation_negg": [0.5616379380226135, 0.5921094417572021, 0.562293291091919], "cos_apical_negg": [0.5616379380226135, 0.5921094417572021, 0.562293291091919], "cos_Ac_negg": [0.5616379380226135, 0.5921094417572021, 0.562293291091919], "r_norm": [0.24694493412971497, 0.24161775410175323, 0.23929674923419952]}, {"step": 1700, "epoch": 4, "train_loss": 0.06270185858011246, "cos_r_negg": [0.5727818012237549, 0.5971183776855469, 0.5600703954696655], "cos_innovation_negg": [0.5727818012237549, 0.5971183776855469, 0.5600703954696655], "cos_apical_negg": [0.5727818012237549, 0.5971183776855469, 0.5600703954696655], "cos_Ac_negg": [0.5727818012237549, 0.5971183776855469, 0.5600703954696655], "r_norm": [0.22281652688980103, 0.21695896983146667, 0.21679364144802094]}, {"step": 1750, "epoch": 4, "train_loss": 0.06852933019399643, "cos_r_negg": [0.574419379234314, 0.606897234916687, 0.5730010867118835], "cos_innovation_negg": [0.574419379234314, 0.606897234916687, 0.5730010867118835], "cos_apical_negg": [0.574419379234314, 0.606897234916687, 0.5730010867118835], "cos_Ac_negg": [0.574419379234314, 0.606897234916687, 0.5730010867118835], "r_norm": [0.22250446677207947, 0.21605323255062103, 0.21634125709533691], "ldr": {"dL_sdil": -0.005061499774456024, "dL_bp": -0.005225874483585358, "ratio": 0.968545989834689}}, {"step": 1800, "epoch": 4, "train_loss": 0.030726507306098938, "cos_r_negg": [0.5594089031219482, 0.5999141931533813, 0.5654407739639282], "cos_innovation_negg": [0.5594089031219482, 0.5999141931533813, 0.5654407739639282], "cos_apical_negg": [0.5594089031219482, 0.5999141931533813, 0.5654407739639282], "cos_Ac_negg": [0.5594089031219482, 0.5999141931533813, 0.5654407739639282], "r_norm": [0.23108433187007904, 0.2256322056055069, 0.22614920139312744]}, {"step": 1850, "epoch": 4, "train_loss": 0.23883147537708282, "cos_r_negg": [0.5604827404022217, 0.6024421453475952, 0.5729967951774597], "cos_innovation_negg": [0.5604827404022217, 0.6024421453475952, 0.5729967951774597], "cos_apical_negg": [0.5604827404022217, 0.6024421453475952, 0.5729967951774597], "cos_Ac_negg": [0.5604827404022217, 0.6024421453475952, 0.5729967951774597], "r_norm": [0.23960557579994202, 0.23245276510715485, 0.23442839086055756]}, {"step": 1900, "epoch": 4, "train_loss": 0.011738545261323452, "cos_r_negg": [0.5867949724197388, 0.6116671562194824, 0.5823869705200195], "cos_innovation_negg": [0.5867949724197388, 0.6116671562194824, 0.5823869705200195], "cos_apical_negg": [0.5867949724197388, 0.6116671562194824, 0.5823869705200195], "cos_Ac_negg": [0.5867949724197388, 0.6116671562194824, 0.5823869705200195], "r_norm": [0.21083800494670868, 0.20459392666816711, 0.20441386103630066]}, {"step": 1950, "epoch": 4, "train_loss": 0.09043135493993759, "cos_r_negg": [0.5733475685119629, 0.6115942597389221, 0.5846699476242065], "cos_innovation_negg": [0.5733475685119629, 0.6115942597389221, 0.5846699476242065], "cos_apical_negg": [0.5733475685119629, 0.6115942597389221, 0.5846699476242065], "cos_Ac_negg": [0.5733475685119629, 0.6115942597389221, 0.5846699476242065], "r_norm": [0.21319063007831573, 0.2065821886062622, 0.20721781253814697]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.9653, "test_loss": 0.1206702980041504}, {"step": 2000, "epoch": 5, "train_loss": 0.03136630728840828, "cos_r_negg": [0.5794245600700378, 0.6152697801589966, 0.591656506061554], "cos_innovation_negg": [0.5794245600700378, 0.6152697801589966, 0.591656506061554], "cos_apical_negg": [0.5794245600700378, 0.6152697801589966, 0.591656506061554], "cos_Ac_negg": [0.5794245600700378, 0.6152697801589966, 0.591656506061554], "r_norm": [0.21051433682441711, 0.20554722845554352, 0.20458292961120605], "ldr": {"dL_sdil": -0.005520857870578766, "dL_bp": -0.006051808595657349, "ratio": 0.912265776967966}}, {"step": 2050, "epoch": 5, "train_loss": 0.008673222735524178, "cos_r_negg": [0.5676900148391724, 0.6171913146972656, 0.5914634466171265], "cos_innovation_negg": [0.5676900148391724, 0.6171913146972656, 0.5914634466171265], "cos_apical_negg": [0.5676900148391724, 0.6171913146972656, 0.5914634466171265], "cos_Ac_negg": [0.5676900148391724, 0.6171913146972656, 0.5914634466171265], "r_norm": [0.2198503017425537, 0.21371416747570038, 0.21302101016044617]}, {"step": 2100, "epoch": 5, "train_loss": 0.04614477604627609, "cos_r_negg": [0.586115837097168, 0.6234146952629089, 0.5998377799987793], "cos_innovation_negg": [0.586115837097168, 0.6234146952629089, 0.5998377799987793], "cos_apical_negg": [0.586115837097168, 0.6234146952629089, 0.5998377799987793], "cos_Ac_negg": [0.586115837097168, 0.6234146952629089, 0.5998377799987793], "r_norm": [0.23148825764656067, 0.2235516458749771, 0.22307837009429932]}, {"step": 2150, "epoch": 5, "train_loss": 0.008139006793498993, "cos_r_negg": [0.5630389451980591, 0.6151437163352966, 0.588782548904419], "cos_innovation_negg": [0.5630389451980591, 0.6151437163352966, 0.588782548904419], "cos_apical_negg": [0.5630389451980591, 0.6151437163352966, 0.588782548904419], "cos_Ac_negg": [0.5630389451980591, 0.6151437163352966, 0.588782548904419], "r_norm": [0.2212124615907669, 0.2147892415523529, 0.2140892744064331]}, {"step": 2200, "epoch": 5, "train_loss": 0.021328624337911606, "cos_r_negg": [0.5817954540252686, 0.6209361553192139, 0.5870919823646545], "cos_innovation_negg": [0.5817954540252686, 0.6209361553192139, 0.5870919823646545], "cos_apical_negg": [0.5817954540252686, 0.6209361553192139, 0.5870919823646545], "cos_Ac_negg": [0.5817954540252686, 0.6209361553192139, 0.5870919823646545], "r_norm": [0.19517794251441956, 0.18972191214561462, 0.19064801931381226]}, {"step": 2250, "epoch": 5, "train_loss": 0.007710002362728119, "cos_r_negg": [0.5794533491134644, 0.6252869367599487, 0.5960915088653564], "cos_innovation_negg": [0.5794533491134644, 0.6252869367599487, 0.5960915088653564], "cos_apical_negg": [0.5794533491134644, 0.6252869367599487, 0.5960915088653564], "cos_Ac_negg": [0.5794533491134644, 0.6252869367599487, 0.5960915088653564], "r_norm": [0.19378358125686646, 0.18594136834144592, 0.1865391731262207], "ldr": {"dL_sdil": -0.005242139101028442, "dL_bp": -0.006118476390838623, "ratio": 0.8567719749442285}}, {"step": 2300, "epoch": 5, "train_loss": 0.03437408059835434, "cos_r_negg": [0.5878440737724304, 0.6275931596755981, 0.6014403104782104], "cos_innovation_negg": [0.5878440737724304, 0.6275931596755981, 0.6014403104782104], "cos_apical_negg": [0.5878440737724304, 0.6275931596755981, 0.6014403104782104], "cos_Ac_negg": [0.5878440737724304, 0.6275931596755981, 0.6014403104782104], "r_norm": [0.19298750162124634, 0.1887616515159607, 0.18750958144664764]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9669, "test_loss": 0.1244256378173828}, {"step": 2350, "epoch": 6, "train_loss": 0.010994862765073776, "cos_r_negg": [0.5706233978271484, 0.6243610382080078, 0.6048367619514465], "cos_innovation_negg": [0.5706233978271484, 0.6243610382080078, 0.6048367619514465], "cos_apical_negg": [0.5706233978271484, 0.6243610382080078, 0.6048367619514465], "cos_Ac_negg": [0.5706233978271484, 0.6243610382080078, 0.6048367619514465], "r_norm": [0.19175666570663452, 0.18709591031074524, 0.18684163689613342]}, {"step": 2400, "epoch": 6, "train_loss": 0.009705914184451103, "cos_r_negg": [0.5862320065498352, 0.624199390411377, 0.61173415184021], "cos_innovation_negg": [0.5862320065498352, 0.624199390411377, 0.61173415184021], "cos_apical_negg": [0.5862320065498352, 0.624199390411377, 0.61173415184021], "cos_Ac_negg": [0.5862320065498352, 0.624199390411377, 0.61173415184021], "r_norm": [0.21907305717468262, 0.21457268297672272, 0.21322283148765564]}, {"step": 2450, "epoch": 6, "train_loss": 0.01840166002511978, "cos_r_negg": [0.5886368751525879, 0.6308690309524536, 0.6081572771072388], "cos_innovation_negg": [0.5886368751525879, 0.6308690309524536, 0.6081572771072388], "cos_apical_negg": [0.5886368751525879, 0.6308690309524536, 0.6081572771072388], "cos_Ac_negg": [0.5886368751525879, 0.6308690309524536, 0.6081572771072388], "r_norm": [0.18867036700248718, 0.1834287941455841, 0.18319803476333618]}, {"step": 2500, "epoch": 6, "train_loss": 0.007425863761454821, "cos_r_negg": [0.6043969988822937, 0.6417964696884155, 0.6164345145225525], "cos_innovation_negg": [0.6043969988822937, 0.6417964696884155, 0.6164345145225525], "cos_apical_negg": [0.6043969988822937, 0.6417964696884155, 0.6164345145225525], "cos_Ac_negg": [0.6043969988822937, 0.6417964696884155, 0.6164345145225525], "r_norm": [0.18309661746025085, 0.1786317229270935, 0.17827796936035156], "ldr": {"dL_sdil": -0.0054972246289253235, "dL_bp": -0.006397902965545654, "ratio": 0.8592228826428419}}, {"step": 2550, "epoch": 6, "train_loss": 0.012249086052179337, "cos_r_negg": [0.5971106290817261, 0.6324653625488281, 0.6123660206794739], "cos_innovation_negg": [0.5971106290817261, 0.6324653625488281, 0.6123660206794739], "cos_apical_negg": [0.5971106290817261, 0.6324653625488281, 0.6123660206794739], "cos_Ac_negg": [0.5971106290817261, 0.6324653625488281, 0.6123660206794739], "r_norm": [0.1685314178466797, 0.16358397901058197, 0.16297538578510284]}, {"step": 2600, "epoch": 6, "train_loss": 0.02514684945344925, "cos_r_negg": [0.5860853791236877, 0.6343705654144287, 0.6182618737220764], "cos_innovation_negg": [0.5860853791236877, 0.6343705654144287, 0.6182618737220764], "cos_apical_negg": [0.5860853791236877, 0.6343705654144287, 0.6182618737220764], "cos_Ac_negg": [0.5860853791236877, 0.6343705654144287, 0.6182618737220764], "r_norm": [0.15029923617839813, 0.14692367613315582, 0.1451931893825531]}, {"step": 2650, "epoch": 6, "train_loss": 0.037182651460170746, "cos_r_negg": [0.5846997499465942, 0.6346275806427002, 0.6169319152832031], "cos_innovation_negg": [0.5846997499465942, 0.6346275806427002, 0.6169319152832031], "cos_apical_negg": [0.5846997499465942, 0.6346275806427002, 0.6169319152832031], "cos_Ac_negg": [0.5846997499465942, 0.6346275806427002, 0.6169319152832031], "r_norm": [0.17399358749389648, 0.16871437430381775, 0.16888996958732605]}, {"step": 2700, "epoch": 6, "train_loss": 0.004268167074769735, "cos_r_negg": [0.5845916271209717, 0.6320409774780273, 0.6193678379058838], "cos_innovation_negg": [0.5845916271209717, 0.6320409774780273, 0.6193678379058838], "cos_apical_negg": [0.5845916271209717, 0.6320409774780273, 0.6193678379058838], "cos_Ac_negg": [0.5845916271209717, 0.6320409774780273, 0.6193678379058838], "r_norm": [0.19429850578308105, 0.18789151310920715, 0.1864216923713684]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9672, "test_loss": 0.12733048210144043}, {"step": 2750, "epoch": 7, "train_loss": 0.0011055364739149809, "cos_r_negg": [0.5960479974746704, 0.6390166282653809, 0.6245473027229309], "cos_innovation_negg": [0.5960479974746704, 0.6390166282653809, 0.6245473027229309], "cos_apical_negg": [0.5960479974746704, 0.6390166282653809, 0.6245473027229309], "cos_Ac_negg": [0.5960479974746704, 0.6390166282653809, 0.6245473027229309], "r_norm": [0.16487577557563782, 0.1602708101272583, 0.15986640751361847], "ldr": {"dL_sdil": -0.004281565546989441, "dL_bp": -0.0052613765001297, "ratio": 0.8137728875483241}}, {"step": 2800, "epoch": 7, "train_loss": 0.0021975371055305004, "cos_r_negg": [0.5993116497993469, 0.6437944173812866, 0.6322163939476013], "cos_innovation_negg": [0.5993116497993469, 0.6437944173812866, 0.6322163939476013], "cos_apical_negg": [0.5993116497993469, 0.6437944173812866, 0.6322163939476013], "cos_Ac_negg": [0.5993116497993469, 0.6437944173812866, 0.6322163939476013], "r_norm": [0.1689635068178177, 0.16407859325408936, 0.16418665647506714]}, {"step": 2850, "epoch": 7, "train_loss": 0.0034289348404854536, "cos_r_negg": [0.5885773301124573, 0.6390666365623474, 0.6307986974716187], "cos_innovation_negg": [0.5885773301124573, 0.6390666365623474, 0.6307986974716187], "cos_apical_negg": [0.5885773301124573, 0.6390666365623474, 0.6307986974716187], "cos_Ac_negg": [0.5885773301124573, 0.6390666365623474, 0.6307986974716187], "r_norm": [0.17239691317081451, 0.16479641199111938, 0.16562937200069427]}, {"step": 2900, "epoch": 7, "train_loss": 0.009095774032175541, "cos_r_negg": [0.587206244468689, 0.646519660949707, 0.6348845958709717], "cos_innovation_negg": [0.587206244468689, 0.646519660949707, 0.6348845958709717], "cos_apical_negg": [0.587206244468689, 0.646519660949707, 0.6348845958709717], "cos_Ac_negg": [0.587206244468689, 0.646519660949707, 0.6348845958709717], "r_norm": [0.1650846004486084, 0.15876959264278412, 0.1588941514492035]}, {"step": 2950, "epoch": 7, "train_loss": 0.006902494002133608, "cos_r_negg": [0.5874362587928772, 0.6450093984603882, 0.6390566825866699], "cos_innovation_negg": [0.5874362587928772, 0.6450093984603882, 0.6390566825866699], "cos_apical_negg": [0.5874362587928772, 0.6450093984603882, 0.6390566825866699], "cos_Ac_negg": [0.5874362587928772, 0.6450093984603882, 0.6390566825866699], "r_norm": [0.17950314283370972, 0.17402714490890503, 0.1743157058954239]}, {"step": 3000, "epoch": 7, "train_loss": 0.004376638680696487, "cos_r_negg": [0.5938992500305176, 0.649909257888794, 0.6423361301422119], "cos_innovation_negg": [0.5938992500305176, 0.649909257888794, 0.6423361301422119], "cos_apical_negg": [0.5938992500305176, 0.649909257888794, 0.6423361301422119], "cos_Ac_negg": [0.5938992500305176, 0.649909257888794, 0.6423361301422119], "r_norm": [0.15046323835849762, 0.14636968076229095, 0.1462033987045288], "ldr": {"dL_sdil": -0.004428751766681671, "dL_bp": -0.006209827959537506, "ratio": 0.7131842935970024}}, {"step": 3050, "epoch": 7, "train_loss": 0.005441409535706043, "cos_r_negg": [0.6072732210159302, 0.6452292203903198, 0.6452909708023071], "cos_innovation_negg": [0.6072732210159302, 0.6452292203903198, 0.6452909708023071], "cos_apical_negg": [0.6072732210159302, 0.6452292203903198, 0.6452909708023071], "cos_Ac_negg": [0.6072732210159302, 0.6452292203903198, 0.6452909708023071], "r_norm": [0.1636664867401123, 0.15956896543502808, 0.15888163447380066]}, {"step": 3100, "epoch": 7, "train_loss": 0.0014721904881298542, "cos_r_negg": [0.6022357940673828, 0.6459878087043762, 0.6450864672660828], "cos_innovation_negg": [0.6022357940673828, 0.6459878087043762, 0.6450864672660828], "cos_apical_negg": [0.6022357940673828, 0.6459878087043762, 0.6450864672660828], "cos_Ac_negg": [0.6022357940673828, 0.6459878087043762, 0.6450864672660828], "r_norm": [0.15064328908920288, 0.14617256820201874, 0.14516222476959229]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9686, "test_loss": 0.13148560485839844}, {"step": 3150, "epoch": 8, "train_loss": 0.008206404745578766, "cos_r_negg": [0.6023980975151062, 0.6414042711257935, 0.6450884342193604], "cos_innovation_negg": [0.6023980975151062, 0.6414042711257935, 0.6450884342193604], "cos_apical_negg": [0.6023980975151062, 0.6414042711257935, 0.6450884342193604], "cos_Ac_negg": [0.6023980975151062, 0.6414042711257935, 0.6450884342193604], "r_norm": [0.1929910033941269, 0.18790489435195923, 0.18641211092472076]}, {"step": 3200, "epoch": 8, "train_loss": 0.02432141825556755, "cos_r_negg": [0.6008176803588867, 0.6489701271057129, 0.6499689817428589], "cos_innovation_negg": [0.6008176803588867, 0.6489701271057129, 0.6499689817428589], "cos_apical_negg": [0.6008176803588867, 0.6489701271057129, 0.6499689817428589], "cos_Ac_negg": [0.6008176803588867, 0.6489701271057129, 0.6499689817428589], "r_norm": [0.1693996787071228, 0.16469818353652954, 0.16444990038871765]}, {"step": 3250, "epoch": 8, "train_loss": 0.0022876248694956303, "cos_r_negg": [0.6062471866607666, 0.6497995853424072, 0.6491546630859375], "cos_innovation_negg": [0.6062471866607666, 0.6497995853424072, 0.6491546630859375], "cos_apical_negg": [0.6062471866607666, 0.6497995853424072, 0.6491546630859375], "cos_Ac_negg": [0.6062471866607666, 0.6497995853424072, 0.6491546630859375], "r_norm": [0.16671639680862427, 0.16175463795661926, 0.16060498356819153], "ldr": {"dL_sdil": -0.0056724101305007935, "dL_bp": -0.00756053626537323, "ratio": 0.750265580562065}}, {"step": 3300, "epoch": 8, "train_loss": 0.0032197656109929085, "cos_r_negg": [0.5904723405838013, 0.6428415775299072, 0.6429845094680786], "cos_innovation_negg": [0.5904723405838013, 0.6428415775299072, 0.6429845094680786], "cos_apical_negg": [0.5904723405838013, 0.6428415775299072, 0.6429845094680786], "cos_Ac_negg": [0.5904723405838013, 0.6428415775299072, 0.6429845094680786], "r_norm": [0.1425301432609558, 0.1382196843624115, 0.13801033794879913]}, {"step": 3350, "epoch": 8, "train_loss": 0.01919647678732872, "cos_r_negg": [0.5929849147796631, 0.6499567627906799, 0.6510373950004578], "cos_innovation_negg": [0.5929849147796631, 0.6499567627906799, 0.6510373950004578], "cos_apical_negg": [0.5929849147796631, 0.6499567627906799, 0.6510373950004578], "cos_Ac_negg": [0.5929849147796631, 0.6499567627906799, 0.6510373950004578], "r_norm": [0.15364235639572144, 0.14933368563652039, 0.14844371378421783]}, {"step": 3400, "epoch": 8, "train_loss": 0.0019633565098047256, "cos_r_negg": [0.5825097560882568, 0.64826899766922, 0.6512289047241211], "cos_innovation_negg": [0.5825097560882568, 0.64826899766922, 0.6512289047241211], "cos_apical_negg": [0.5825097560882568, 0.64826899766922, 0.6512289047241211], "cos_Ac_negg": [0.5825097560882568, 0.64826899766922, 0.6512289047241211], "r_norm": [0.15885940194129944, 0.15482422709465027, 0.15351760387420654]}, {"step": 3450, "epoch": 8, "train_loss": 0.01184171624481678, "cos_r_negg": [0.5965076684951782, 0.6582930088043213, 0.6605890989303589], "cos_innovation_negg": [0.5965076684951782, 0.6582930088043213, 0.6605890989303589], "cos_apical_negg": [0.5965076684951782, 0.6582930088043213, 0.6605890989303589], "cos_Ac_negg": [0.5965076684951782, 0.6582930088043213, 0.6605890989303589], "r_norm": [0.16533662378787994, 0.160286545753479, 0.1598929464817047]}, {"step": 3500, "epoch": 8, "train_loss": 0.010087580420076847, "cos_r_negg": [0.5935340523719788, 0.6559990644454956, 0.6548561453819275], "cos_innovation_negg": [0.5935340523719788, 0.6559990644454956, 0.6548561453819275], "cos_apical_negg": [0.5935340523719788, 0.6559990644454956, 0.6548561453819275], "cos_Ac_negg": [0.5935340523719788, 0.6559990644454956, 0.6548561453819275], "r_norm": [0.16610638797283173, 0.16042372584342957, 0.1595354676246643], "ldr": {"dL_sdil": -0.006401047110557556, "dL_bp": -0.009617671370506287, "ratio": 0.6655506165647451}}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9705, "test_loss": 0.13136551399230956}, {"step": 3550, "epoch": 9, "train_loss": 0.0022519254125654697, "cos_r_negg": [0.5882115364074707, 0.6448885798454285, 0.6527107357978821], "cos_innovation_negg": [0.5882115364074707, 0.6448885798454285, 0.6527107357978821], "cos_apical_negg": [0.5882115364074707, 0.6448885798454285, 0.6527107357978821], "cos_Ac_negg": [0.5882115364074707, 0.6448885798454285, 0.6527107357978821], "r_norm": [0.1943812370300293, 0.18914780020713806, 0.18675313889980316]}, {"step": 3600, "epoch": 9, "train_loss": 0.0037287052255123854, "cos_r_negg": [0.5874994397163391, 0.6492916345596313, 0.6514467000961304], "cos_innovation_negg": [0.5874994397163391, 0.6492916345596313, 0.6514467000961304], "cos_apical_negg": [0.5874994397163391, 0.6492916345596313, 0.6514467000961304], "cos_Ac_negg": [0.5874994397163391, 0.6492916345596313, 0.6514467000961304], "r_norm": [0.16537204384803772, 0.1602584421634674, 0.15922316908836365]}, {"step": 3650, "epoch": 9, "train_loss": 0.002016789512708783, "cos_r_negg": [0.5953986048698425, 0.6527057886123657, 0.6587442755699158], "cos_innovation_negg": [0.5953986048698425, 0.6527057886123657, 0.6587442755699158], "cos_apical_negg": [0.5953986048698425, 0.6527057886123657, 0.6587442755699158], "cos_Ac_negg": [0.5953986048698425, 0.6527057886123657, 0.6587442755699158], "r_norm": [0.16086316108703613, 0.15631097555160522, 0.1548996865749359]}, {"step": 3700, "epoch": 9, "train_loss": 0.008451085537672043, "cos_r_negg": [0.5814222097396851, 0.6462743282318115, 0.6586074829101562], "cos_innovation_negg": [0.5814222097396851, 0.6462743282318115, 0.6586074829101562], "cos_apical_negg": [0.5814222097396851, 0.6462743282318115, 0.6586074829101562], "cos_Ac_negg": [0.5814222097396851, 0.6462743282318115, 0.6586074829101562], "r_norm": [0.15844686329364777, 0.15337729454040527, 0.15336091816425323]}, {"step": 3750, "epoch": 9, "train_loss": 0.006918740924447775, "cos_r_negg": [0.5869224667549133, 0.6502401828765869, 0.6606221795082092], "cos_innovation_negg": [0.5869224667549133, 0.6502401828765869, 0.6606221795082092], "cos_apical_negg": [0.5869224667549133, 0.6502401828765869, 0.6606221795082092], "cos_Ac_negg": [0.5869224667549133, 0.6502401828765869, 0.6606221795082092], "r_norm": [0.15791988372802734, 0.15279217064380646, 0.15181350708007812], "ldr": {"dL_sdil": -0.005951322615146637, "dL_bp": -0.009503483772277832, "ratio": 0.6262253672181733}}, {"step": 3800, "epoch": 9, "train_loss": 0.0010650937911123037, "cos_r_negg": [0.5801271200180054, 0.6482824087142944, 0.6626810431480408], "cos_innovation_negg": [0.5801271200180054, 0.6482824087142944, 0.6626810431480408], "cos_apical_negg": [0.5801271200180054, 0.6482824087142944, 0.6626810431480408], "cos_Ac_negg": [0.5801271200180054, 0.6482824087142944, 0.6626810431480408], "r_norm": [0.14452235400676727, 0.14032159745693207, 0.13960891962051392]}, {"step": 3850, "epoch": 9, "train_loss": 0.0025850138626992702, "cos_r_negg": [0.5903715491294861, 0.6511387228965759, 0.666958749294281], "cos_innovation_negg": [0.5903715491294861, 0.6511387228965759, 0.666958749294281], "cos_apical_negg": [0.5903715491294861, 0.6511387228965759, 0.666958749294281], "cos_Ac_negg": [0.5903715491294861, 0.6511387228965759, 0.666958749294281], "r_norm": [0.1484842449426651, 0.14440935850143433, 0.14332400262355804]}, {"step": 3900, "epoch": 9, "train_loss": 0.006497877184301615, "cos_r_negg": [0.5829071998596191, 0.6471721529960632, 0.6617736220359802], "cos_innovation_negg": [0.5829071998596191, 0.6471721529960632, 0.6617736220359802], "cos_apical_negg": [0.5829071998596191, 0.6471721529960632, 0.6617736220359802], "cos_Ac_negg": [0.5829071998596191, 0.6471721529960632, 0.6617736220359802], "r_norm": [0.13953560590744019, 0.13510355353355408, 0.13490425050258636]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9713, "test_loss": 0.13539419746398926}, {"step": 3950, "epoch": 10, "train_loss": 0.0016516181640326977, "cos_r_negg": [0.5875574350357056, 0.6488780975341797, 0.6622357368469238], "cos_innovation_negg": [0.5875574350357056, 0.6488780975341797, 0.6622357368469238], "cos_apical_negg": [0.5875574350357056, 0.6488780975341797, 0.6622357368469238], "cos_Ac_negg": [0.5875574350357056, 0.6488780975341797, 0.6622357368469238], "r_norm": [0.15231919288635254, 0.14735756814479828, 0.14744895696640015]}, {"step": 4000, "epoch": 10, "train_loss": 0.0020416397601366043, "cos_r_negg": [0.5853614807128906, 0.646879255771637, 0.6637753844261169], "cos_innovation_negg": [0.5853614807128906, 0.646879255771637, 0.6637753844261169], "cos_apical_negg": [0.5853614807128906, 0.646879255771637, 0.6637753844261169], "cos_Ac_negg": [0.5853614807128906, 0.646879255771637, 0.6637753844261169], "r_norm": [0.16070829331874847, 0.15605497360229492, 0.1550750434398651], "ldr": {"dL_sdil": -0.0065033212304115295, "dL_bp": -0.010506764054298401, "ratio": 0.6189651920232252}}, {"step": 4050, "epoch": 10, "train_loss": 0.0014407174894586205, "cos_r_negg": [0.5838597416877747, 0.6469875574111938, 0.6645874381065369], "cos_innovation_negg": [0.5838597416877747, 0.6469875574111938, 0.6645874381065369], "cos_apical_negg": [0.5838597416877747, 0.6469875574111938, 0.6645874381065369], "cos_Ac_negg": [0.5838597416877747, 0.6469875574111938, 0.6645874381065369], "r_norm": [0.1562536060810089, 0.1514674425125122, 0.1515086591243744]}, {"step": 4100, "epoch": 10, "train_loss": 0.0010526756523177028, "cos_r_negg": [0.5852677226066589, 0.6494618654251099, 0.6648362278938293], "cos_innovation_negg": [0.5852677226066589, 0.6494618654251099, 0.6648362278938293], "cos_apical_negg": [0.5852677226066589, 0.6494618654251099, 0.6648362278938293], "cos_Ac_negg": [0.5852677226066589, 0.6494618654251099, 0.6648362278938293], "r_norm": [0.15417805314064026, 0.14922910928726196, 0.14903894066810608]}, {"step": 4150, "epoch": 10, "train_loss": 0.0008821700466796756, "cos_r_negg": [0.5866588354110718, 0.6490439176559448, 0.6645362973213196], "cos_innovation_negg": [0.5866588354110718, 0.6490439176559448, 0.6645362973213196], "cos_apical_negg": [0.5866588354110718, 0.6490439176559448, 0.6645362973213196], "cos_Ac_negg": [0.5866588354110718, 0.6490439176559448, 0.6645362973213196], "r_norm": [0.15289895236492157, 0.14816167950630188, 0.14714491367340088]}, {"step": 4200, "epoch": 10, "train_loss": 0.002147999592125416, "cos_r_negg": [0.5856039524078369, 0.6515648365020752, 0.6653529405593872], "cos_innovation_negg": [0.5856039524078369, 0.6515648365020752, 0.6653529405593872], "cos_apical_negg": [0.5856039524078369, 0.6515648365020752, 0.6653529405593872], "cos_Ac_negg": [0.5856039524078369, 0.6515648365020752, 0.6653529405593872], "r_norm": [0.15915261209011078, 0.15448741614818573, 0.15314733982086182]}, {"step": 4250, "epoch": 10, "train_loss": 0.0056985593400895596, "cos_r_negg": [0.5824014544487, 0.6472827196121216, 0.6624464988708496], "cos_innovation_negg": [0.5824014544487, 0.6472827196121216, 0.6624464988708496], "cos_apical_negg": [0.5824014544487, 0.6472827196121216, 0.6624464988708496], "cos_Ac_negg": [0.5824014544487, 0.6472827196121216, 0.6624464988708496], "r_norm": [0.16633158922195435, 0.1617356687784195, 0.16036450862884521], "ldr": {"dL_sdil": -0.007304109632968903, "dL_bp": -0.012216918170452118, "ratio": 0.5978684256586615}}, {"step": 4300, "epoch": 10, "train_loss": 0.0033146252389997244, "cos_r_negg": [0.5766094923019409, 0.644221842288971, 0.663758397102356], "cos_innovation_negg": [0.5766094923019409, 0.644221842288971, 0.663758397102356], "cos_apical_negg": [0.5766094923019409, 0.644221842288971, 0.663758397102356], "cos_Ac_negg": [0.5766094923019409, 0.644221842288971, 0.663758397102356], "r_norm": [0.16306915879249573, 0.15827858448028564, 0.15692895650863647]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9715, "test_loss": 0.1414761137008667}, {"step": 4350, "epoch": 11, "train_loss": 0.0018337933579459786, "cos_r_negg": [0.5778127908706665, 0.6457934379577637, 0.6628637313842773], "cos_innovation_negg": [0.5778127908706665, 0.6457934379577637, 0.6628637313842773], "cos_apical_negg": [0.5778127908706665, 0.6457934379577637, 0.6628637313842773], "cos_Ac_negg": [0.5778127908706665, 0.6457934379577637, 0.6628637313842773], "r_norm": [0.15650677680969238, 0.15211842954158783, 0.15094023942947388]}, {"step": 4400, "epoch": 11, "train_loss": 0.0004373940173536539, "cos_r_negg": [0.5772349834442139, 0.6457128524780273, 0.663483202457428], "cos_innovation_negg": [0.5772349834442139, 0.6457128524780273, 0.663483202457428], "cos_apical_negg": [0.5772349834442139, 0.6457128524780273, 0.663483202457428], "cos_Ac_negg": [0.5772349834442139, 0.6457128524780273, 0.663483202457428], "r_norm": [0.153253436088562, 0.14894244074821472, 0.14769871532917023]}, {"step": 4450, "epoch": 11, "train_loss": 0.0040444303303956985, "cos_r_negg": [0.5771080255508423, 0.6459321975708008, 0.66600102186203], "cos_innovation_negg": [0.5771080255508423, 0.6459321975708008, 0.66600102186203], "cos_apical_negg": [0.5771080255508423, 0.6459321975708008, 0.66600102186203], "cos_Ac_negg": [0.5771080255508423, 0.6459321975708008, 0.66600102186203], "r_norm": [0.15830761194229126, 0.1537318229675293, 0.1524672657251358]}, {"step": 4500, "epoch": 11, "train_loss": 0.0006421637372113764, "cos_r_negg": [0.578391969203949, 0.6479247808456421, 0.6655681729316711], "cos_innovation_negg": [0.578391969203949, 0.6479247808456421, 0.6655681729316711], "cos_apical_negg": [0.578391969203949, 0.6479247808456421, 0.6655681729316711], "cos_Ac_negg": [0.578391969203949, 0.6479247808456421, 0.6655681729316711], "r_norm": [0.15463458001613617, 0.15012452006340027, 0.14921589195728302], "ldr": {"dL_sdil": -0.006584614515304565, "dL_bp": -0.011204428970813751, "ratio": 0.5876796160211939}}, {"step": 4550, "epoch": 11, "train_loss": 0.0036143872421234846, "cos_r_negg": [0.5753625631332397, 0.6456612348556519, 0.6666555404663086], "cos_innovation_negg": [0.5753625631332397, 0.6456612348556519, 0.6666555404663086], "cos_apical_negg": [0.5753625631332397, 0.6456612348556519, 0.6666555404663086], "cos_Ac_negg": [0.5753625631332397, 0.6456612348556519, 0.6666555404663086], "r_norm": [0.15706199407577515, 0.15249688923358917, 0.15135611593723297]}, {"step": 4600, "epoch": 11, "train_loss": 0.0003669715952128172, "cos_r_negg": [0.5731289386749268, 0.6455035209655762, 0.6664634943008423], "cos_innovation_negg": [0.5731289386749268, 0.6455035209655762, 0.6664634943008423], "cos_apical_negg": [0.5731289386749268, 0.6455035209655762, 0.6664634943008423], "cos_Ac_negg": [0.5731289386749268, 0.6455035209655762, 0.6664634943008423], "r_norm": [0.1547778993844986, 0.15015967190265656, 0.14900997281074524]}, {"step": 4650, "epoch": 11, "train_loss": 0.0009130017133429646, "cos_r_negg": [0.5750152468681335, 0.6501655578613281, 0.6715781092643738], "cos_innovation_negg": [0.5750152468681335, 0.6501655578613281, 0.6715781092643738], "cos_apical_negg": [0.5750152468681335, 0.6501655578613281, 0.6715781092643738], "cos_Ac_negg": [0.5750152468681335, 0.6501655578613281, 0.6715781092643738], "r_norm": [0.1539333164691925, 0.1494208425283432, 0.1482224464416504]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9709, "test_loss": 0.145171950340271}, {"step": 4700, "epoch": 12, "train_loss": 0.0022361362352967262, "cos_r_negg": [0.5681445002555847, 0.6467467546463013, 0.6679610013961792], "cos_innovation_negg": [0.5681445002555847, 0.6467467546463013, 0.6679610013961792], "cos_apical_negg": [0.5681445002555847, 0.6467467546463013, 0.6679610013961792], "cos_Ac_negg": [0.5681445002555847, 0.6467467546463013, 0.6679610013961792], "r_norm": [0.1525803506374359, 0.14821189641952515, 0.1472267508506775]}, {"step": 4750, "epoch": 12, "train_loss": 0.0006564384093508124, "cos_r_negg": [0.5718768835067749, 0.6469906568527222, 0.6682277917861938], "cos_innovation_negg": [0.5718768835067749, 0.6469906568527222, 0.6682277917861938], "cos_apical_negg": [0.5718768835067749, 0.6469906568527222, 0.6682277917861938], "cos_Ac_negg": [0.5718768835067749, 0.6469906568527222, 0.6682277917861938], "r_norm": [0.1537533700466156, 0.14945191144943237, 0.1482844203710556], "ldr": {"dL_sdil": -0.006823793053627014, "dL_bp": -0.011547163128852844, "ratio": 0.5909497404238131}}, {"step": 4800, "epoch": 12, "train_loss": 0.0006446122424677014, "cos_r_negg": [0.5715514421463013, 0.6470254063606262, 0.667757511138916], "cos_innovation_negg": [0.5715514421463013, 0.6470254063606262, 0.667757511138916], "cos_apical_negg": [0.5715514421463013, 0.6470254063606262, 0.667757511138916], "cos_Ac_negg": [0.5715514421463013, 0.6470254063606262, 0.667757511138916], "r_norm": [0.1491921842098236, 0.14494583010673523, 0.14378061890602112]}, {"step": 4850, "epoch": 12, "train_loss": 0.0010108051355928183, "cos_r_negg": [0.5716310739517212, 0.6471174955368042, 0.6685180068016052], "cos_innovation_negg": [0.5716310739517212, 0.6471174955368042, 0.6685180068016052], "cos_apical_negg": [0.5716310739517212, 0.6471174955368042, 0.6685180068016052], "cos_Ac_negg": [0.5716310739517212, 0.6471174955368042, 0.6685180068016052], "r_norm": [0.15078449249267578, 0.146612286567688, 0.14540421962738037]}, {"step": 4900, "epoch": 12, "train_loss": 0.0009043274912983179, "cos_r_negg": [0.5724226832389832, 0.6471394300460815, 0.6692046523094177], "cos_innovation_negg": [0.5724226832389832, 0.6471394300460815, 0.6692046523094177], "cos_apical_negg": [0.5724226832389832, 0.6471394300460815, 0.6692046523094177], "cos_Ac_negg": [0.5724226832389832, 0.6471394300460815, 0.6692046523094177], "r_norm": [0.1521827131509781, 0.14782044291496277, 0.1469435691833496]}, {"step": 4950, "epoch": 12, "train_loss": 0.00036247915704734623, "cos_r_negg": [0.5707689523696899, 0.6465746164321899, 0.6705124378204346], "cos_innovation_negg": [0.5707689523696899, 0.6465746164321899, 0.6705124378204346], "cos_apical_negg": [0.5707689523696899, 0.6465746164321899, 0.6705124378204346], "cos_Ac_negg": [0.5707689523696899, 0.6465746164321899, 0.6705124378204346], "r_norm": [0.15753820538520813, 0.15288755297660828, 0.15184736251831055]}, {"step": 5000, "epoch": 12, "train_loss": 0.0012834727531298995, "cos_r_negg": [0.573039710521698, 0.6490299105644226, 0.6723425388336182], "cos_innovation_negg": [0.573039710521698, 0.6490299105644226, 0.6723425388336182], "cos_apical_negg": [0.573039710521698, 0.6490299105644226, 0.6723425388336182], "cos_Ac_negg": [0.573039710521698, 0.6490299105644226, 0.6723425388336182], "r_norm": [0.1589983105659485, 0.15432000160217285, 0.15333721041679382], "ldr": {"dL_sdil": -0.007053345441818237, "dL_bp": -0.012128815054893494, "ratio": 0.5815362349822041}}, {"step": 5050, "epoch": 12, "train_loss": 0.0008302669157274067, "cos_r_negg": [0.568706214427948, 0.6471714377403259, 0.6714260578155518], "cos_innovation_negg": [0.568706214427948, 0.6471714377403259, 0.6714260578155518], "cos_apical_negg": [0.568706214427948, 0.6471714377403259, 0.6714260578155518], "cos_Ac_negg": [0.568706214427948, 0.6471714377403259, 0.6714260578155518], "r_norm": [0.15835466980934143, 0.15386788547039032, 0.1529293656349182]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9715, "test_loss": 0.14806803169250488}, {"step": 5100, "epoch": 13, "train_loss": 0.0010428216774016619, "cos_r_negg": [0.5728027820587158, 0.6487473249435425, 0.6723673343658447], "cos_innovation_negg": [0.5728027820587158, 0.6487473249435425, 0.6723673343658447], "cos_apical_negg": [0.5728027820587158, 0.6487473249435425, 0.6723673343658447], "cos_Ac_negg": [0.5728027820587158, 0.6487473249435425, 0.6723673343658447], "r_norm": [0.16049881279468536, 0.15581372380256653, 0.15480288863182068]}, {"step": 5150, "epoch": 13, "train_loss": 0.0006761642871424556, "cos_r_negg": [0.5734413862228394, 0.6486173868179321, 0.6729018092155457], "cos_innovation_negg": [0.5734413862228394, 0.6486173868179321, 0.6729018092155457], "cos_apical_negg": [0.5734413862228394, 0.6486173868179321, 0.6729018092155457], "cos_Ac_negg": [0.5734413862228394, 0.6486173868179321, 0.6729018092155457], "r_norm": [0.15932923555374146, 0.1548399031162262, 0.15380579233169556]}, {"step": 5200, "epoch": 13, "train_loss": 0.0008530806517228484, "cos_r_negg": [0.5719598531723022, 0.6474353075027466, 0.6737923622131348], "cos_innovation_negg": [0.5719598531723022, 0.6474353075027466, 0.6737923622131348], "cos_apical_negg": [0.5719598531723022, 0.6474353075027466, 0.6737923622131348], "cos_Ac_negg": [0.5719598531723022, 0.6474353075027466, 0.6737923622131348], "r_norm": [0.1555447280406952, 0.1513141691684723, 0.15017078816890717]}, {"step": 5250, "epoch": 13, "train_loss": 0.00023118546232581139, "cos_r_negg": [0.569625198841095, 0.6467342376708984, 0.6734771728515625], "cos_innovation_negg": [0.569625198841095, 0.6467342376708984, 0.6734771728515625], "cos_apical_negg": [0.569625198841095, 0.6467342376708984, 0.6734771728515625], "cos_Ac_negg": [0.569625198841095, 0.6467342376708984, 0.6734771728515625], "r_norm": [0.15839126706123352, 0.15404093265533447, 0.15304899215698242], "ldr": {"dL_sdil": -0.00713193416595459, "dL_bp": -0.011963263154029846, "ratio": 0.5961529119713617}}, {"step": 5300, "epoch": 13, "train_loss": 0.001953431172296405, "cos_r_negg": [0.5711805820465088, 0.6471790075302124, 0.6738486289978027], "cos_innovation_negg": [0.5711805820465088, 0.6471790075302124, 0.6738486289978027], "cos_apical_negg": [0.5711805820465088, 0.6471790075302124, 0.6738486289978027], "cos_Ac_negg": [0.5711805820465088, 0.6471790075302124, 0.6738486289978027], "r_norm": [0.15833048522472382, 0.15399406850337982, 0.15293791890144348]}, {"step": 5350, "epoch": 13, "train_loss": 0.0006202986696735024, "cos_r_negg": [0.5731927156448364, 0.6487444639205933, 0.6739774942398071], "cos_innovation_negg": [0.5731927156448364, 0.6487444639205933, 0.6739774942398071], "cos_apical_negg": [0.5731927156448364, 0.6487444639205933, 0.6739774942398071], "cos_Ac_negg": [0.5731927156448364, 0.6487444639205933, 0.6739774942398071], "r_norm": [0.1597190797328949, 0.15533727407455444, 0.15440551936626434]}, {"step": 5400, "epoch": 13, "train_loss": 0.0007057054899632931, "cos_r_negg": [0.5733116865158081, 0.6477429866790771, 0.6757663488388062], "cos_innovation_negg": [0.5733116865158081, 0.6477429866790771, 0.6757663488388062], "cos_apical_negg": [0.5733116865158081, 0.6477429866790771, 0.6757663488388062], "cos_Ac_negg": [0.5733116865158081, 0.6477429866790771, 0.6757663488388062], "r_norm": [0.15465408563613892, 0.1505601406097412, 0.14960826933383942]}, {"step": 5450, "epoch": 13, "train_loss": 0.00021980002929922193, "cos_r_negg": [0.5725792646408081, 0.6475465297698975, 0.6761786937713623], "cos_innovation_negg": [0.5725792646408081, 0.6475465297698975, 0.6761786937713623], "cos_apical_negg": [0.5725792646408081, 0.6475465297698975, 0.6761786937713623], "cos_Ac_negg": [0.5725792646408081, 0.6475465297698975, 0.6761786937713623], "r_norm": [0.15605735778808594, 0.15186980366706848, 0.15081237256526947]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.9709, "test_loss": 0.15175724182128905}, {"step": 5500, "epoch": 14, "train_loss": 0.0003933235420845449, "cos_r_negg": [0.5728116035461426, 0.6477845907211304, 0.6755441427230835], "cos_innovation_negg": [0.5728116035461426, 0.6477845907211304, 0.6755441427230835], "cos_apical_negg": [0.5728116035461426, 0.6477845907211304, 0.6755441427230835], "cos_Ac_negg": [0.5728116035461426, 0.6477845907211304, 0.6755441427230835], "r_norm": [0.15911388397216797, 0.154710590839386, 0.15367881953716278], "ldr": {"dL_sdil": -0.007528550922870636, "dL_bp": -0.013335369527339935, "ratio": 0.5645551034364467}}, {"step": 5550, "epoch": 14, "train_loss": 0.00023018530919216573, "cos_r_negg": [0.5735013484954834, 0.64759761095047, 0.6771939992904663], "cos_innovation_negg": [0.5735013484954834, 0.64759761095047, 0.6771939992904663], "cos_apical_negg": [0.5735013484954834, 0.64759761095047, 0.6771939992904663], "cos_Ac_negg": [0.5735013484954834, 0.64759761095047, 0.6771939992904663], "r_norm": [0.1569933295249939, 0.15270277857780457, 0.15157440304756165]}, {"step": 5600, "epoch": 14, "train_loss": 0.002001376124098897, "cos_r_negg": [0.5734658241271973, 0.6480486392974854, 0.6769169569015503], "cos_innovation_negg": [0.5734658241271973, 0.6480486392974854, 0.6769169569015503], "cos_apical_negg": [0.5734658241271973, 0.6480486392974854, 0.6769169569015503], "cos_Ac_negg": [0.5734658241271973, 0.6480486392974854, 0.6769169569015503], "r_norm": [0.15761977434158325, 0.15325044095516205, 0.15226945281028748]}, {"step": 5650, "epoch": 14, "train_loss": 0.0007772655226290226, "cos_r_negg": [0.5725285410881042, 0.6480826735496521, 0.6769453287124634], "cos_innovation_negg": [0.5725285410881042, 0.6480826735496521, 0.6769453287124634], "cos_apical_negg": [0.5725285410881042, 0.6480826735496521, 0.6769453287124634], "cos_Ac_negg": [0.5725285410881042, 0.6480826735496521, 0.6769453287124634], "r_norm": [0.15955203771591187, 0.155195415019989, 0.15419277548789978]}, {"step": 5700, "epoch": 14, "train_loss": 0.00017768457473721355, "cos_r_negg": [0.5714725255966187, 0.6471372842788696, 0.6763173937797546], "cos_innovation_negg": [0.5714725255966187, 0.6471372842788696, 0.6763173937797546], "cos_apical_negg": [0.5714725255966187, 0.6471372842788696, 0.6763173937797546], "cos_Ac_negg": [0.5714725255966187, 0.6471372842788696, 0.6763173937797546], "r_norm": [0.15733571350574493, 0.1530742347240448, 0.15200850367546082]}, {"step": 5750, "epoch": 14, "train_loss": 0.0007058955961838365, "cos_r_negg": [0.5719918012619019, 0.6481317281723022, 0.6764578819274902], "cos_innovation_negg": [0.5719918012619019, 0.6481317281723022, 0.6764578819274902], "cos_apical_negg": [0.5719918012619019, 0.6481317281723022, 0.6764578819274902], "cos_Ac_negg": [0.5719918012619019, 0.6481317281723022, 0.6764578819274902], "r_norm": [0.15994670987129211, 0.15562209486961365, 0.15448740124702454], "ldr": {"dL_sdil": -0.007447034120559692, "dL_bp": -0.013312503695487976, "ratio": 0.5594014687923599}}, {"step": 5800, "epoch": 14, "train_loss": 0.0016908473335206509, "cos_r_negg": [0.5689291954040527, 0.6466537117958069, 0.6771152019500732], "cos_innovation_negg": [0.5689291954040527, 0.6466537117958069, 0.6771152019500732], "cos_apical_negg": [0.5689291954040527, 0.6466537117958069, 0.6771152019500732], "cos_Ac_negg": [0.5689291954040527, 0.6466537117958069, 0.6771152019500732], "r_norm": [0.15787117183208466, 0.15356692671775818, 0.15247735381126404]}, {"step": 5850, "epoch": 14, "train_loss": 0.0006827896577306092, "cos_r_negg": [0.5699353814125061, 0.647431492805481, 0.6776337027549744], "cos_innovation_negg": [0.5699353814125061, 0.647431492805481, 0.6776337027549744], "cos_apical_negg": [0.5699353814125061, 0.647431492805481, 0.6776337027549744], "cos_Ac_negg": [0.5699353814125061, 0.647431492805481, 0.6776337027549744], "r_norm": [0.15717646479606628, 0.15296193957328796, 0.15184196829795837]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9713, "test_loss": 0.15317902336120606}, {"step": 5900, "epoch": 15, "train_loss": 0.00035164522705599666, "cos_r_negg": [0.571368932723999, 0.6478909254074097, 0.6791743040084839], "cos_innovation_negg": [0.571368932723999, 0.6478909254074097, 0.6791743040084839], "cos_apical_negg": [0.571368932723999, 0.6478909254074097, 0.6791743040084839], "cos_Ac_negg": [0.571368932723999, 0.6478909254074097, 0.6791743040084839], "r_norm": [0.15883557498455048, 0.1545581817626953, 0.1535111963748932]}, {"step": 5950, "epoch": 15, "train_loss": 0.0006400892743840814, "cos_r_negg": [0.5706825852394104, 0.6473729610443115, 0.6794279217720032], "cos_innovation_negg": [0.5706825852394104, 0.6473729610443115, 0.6794279217720032], "cos_apical_negg": [0.5706825852394104, 0.6473729610443115, 0.6794279217720032], "cos_Ac_negg": [0.5706825852394104, 0.6473729610443115, 0.6794279217720032], "r_norm": [0.15908074378967285, 0.15473401546478271, 0.1537865400314331]}, {"step": 6000, "epoch": 15, "train_loss": 0.00041699467692524195, "cos_r_negg": [0.5701012015342712, 0.6471462249755859, 0.6795181632041931], "cos_innovation_negg": [0.5701012015342712, 0.6471462249755859, 0.6795181632041931], "cos_apical_negg": [0.5701012015342712, 0.6471462249755859, 0.6795181632041931], "cos_Ac_negg": [0.5701012015342712, 0.6471462249755859, 0.6795181632041931], "r_norm": [0.1585027426481247, 0.15422508120536804, 0.1532510221004486], "ldr": {"dL_sdil": -0.007437169551849365, "dL_bp": -0.013234958052635193, "ratio": 0.561933745635753}}, {"step": 6050, "epoch": 15, "train_loss": 0.0019535175524652004, "cos_r_negg": [0.5694942474365234, 0.648208737373352, 0.6791059374809265], "cos_innovation_negg": [0.5694942474365234, 0.648208737373352, 0.6791059374809265], "cos_apical_negg": [0.5694942474365234, 0.648208737373352, 0.6791059374809265], "cos_Ac_negg": [0.5694942474365234, 0.648208737373352, 0.6791059374809265], "r_norm": [0.16220447421073914, 0.15789271891117096, 0.1568143665790558]}, {"step": 6100, "epoch": 15, "train_loss": 0.000306816422380507, "cos_r_negg": [0.5680996775627136, 0.6473686695098877, 0.6785680055618286], "cos_innovation_negg": [0.5680996775627136, 0.6473686695098877, 0.6785680055618286], "cos_apical_negg": [0.5680996775627136, 0.6473686695098877, 0.6785680055618286], "cos_Ac_negg": [0.5680996775627136, 0.6473686695098877, 0.6785680055618286], "r_norm": [0.1606287956237793, 0.1563338339328766, 0.1553012579679489]}, {"step": 6150, "epoch": 15, "train_loss": 0.0003024918551091105, "cos_r_negg": [0.5684484243392944, 0.6482243537902832, 0.6793443560600281], "cos_innovation_negg": [0.5684484243392944, 0.6482243537902832, 0.6793443560600281], "cos_apical_negg": [0.5684484243392944, 0.6482243537902832, 0.6793443560600281], "cos_Ac_negg": [0.5684484243392944, 0.6482243537902832, 0.6793443560600281], "r_norm": [0.1576395332813263, 0.15337425470352173, 0.15235787630081177]}, {"step": 6200, "epoch": 15, "train_loss": 0.0003821358550339937, "cos_r_negg": [0.5685487389564514, 0.6490689516067505, 0.679553747177124], "cos_innovation_negg": [0.5685487389564514, 0.6490689516067505, 0.679553747177124], "cos_apical_negg": [0.5685487389564514, 0.6490689516067505, 0.679553747177124], "cos_Ac_negg": [0.5685487389564514, 0.6490689516067505, 0.679553747177124], "r_norm": [0.159250408411026, 0.15495219826698303, 0.15393885970115662]}, {"step": 6250, "epoch": 15, "train_loss": 0.0002759103663265705, "cos_r_negg": [0.5689799785614014, 0.6485349535942078, 0.6797881722450256], "cos_innovation_negg": [0.5689799785614014, 0.6485349535942078, 0.6797881722450256], "cos_apical_negg": [0.5689799785614014, 0.6485349535942078, 0.6797881722450256], "cos_Ac_negg": [0.5689799785614014, 0.6485349535942078, 0.6797881722450256], "r_norm": [0.15853999555110931, 0.15419110655784607, 0.15321627259254456], "ldr": {"dL_sdil": -0.0076431334018707275, "dL_bp": -0.013785794377326965, "ratio": 0.5544209635420889}}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9713, "test_loss": 0.15588152084350587}, {"step": 6300, "epoch": 16, "train_loss": 0.0008140470599755645, "cos_r_negg": [0.5694584846496582, 0.6489220857620239, 0.6802412867546082], "cos_innovation_negg": [0.5694584846496582, 0.6489220857620239, 0.6802412867546082], "cos_apical_negg": [0.5694584846496582, 0.6489220857620239, 0.6802412867546082], "cos_Ac_negg": [0.5694584846496582, 0.6489220857620239, 0.6802412867546082], "r_norm": [0.15792560577392578, 0.15358991920948029, 0.15254288911819458]}, {"step": 6350, "epoch": 16, "train_loss": 0.00020955759100615978, "cos_r_negg": [0.568501889705658, 0.6480435729026794, 0.6794047951698303], "cos_innovation_negg": [0.568501889705658, 0.6480435729026794, 0.6794047951698303], "cos_apical_negg": [0.568501889705658, 0.6480435729026794, 0.6794047951698303], "cos_Ac_negg": [0.568501889705658, 0.6480435729026794, 0.6794047951698303], "r_norm": [0.15852271020412445, 0.15425103902816772, 0.1531549096107483]}, {"step": 6400, "epoch": 16, "train_loss": 0.000756851164624095, "cos_r_negg": [0.5687994956970215, 0.6476739645004272, 0.6807654500007629], "cos_innovation_negg": [0.5687994956970215, 0.6476739645004272, 0.6807654500007629], "cos_apical_negg": [0.5687994956970215, 0.6476739645004272, 0.6807654500007629], "cos_Ac_negg": [0.5687994956970215, 0.6476739645004272, 0.6807654500007629], "r_norm": [0.15768516063690186, 0.15344437956809998, 0.1524152159690857]}, {"step": 6450, "epoch": 16, "train_loss": 0.00036786601413041353, "cos_r_negg": [0.5704573392868042, 0.6481225490570068, 0.6813298463821411], "cos_innovation_negg": [0.5704573392868042, 0.6481225490570068, 0.6813298463821411], "cos_apical_negg": [0.5704573392868042, 0.6481225490570068, 0.6813298463821411], "cos_Ac_negg": [0.5704573392868042, 0.6481225490570068, 0.6813298463821411], "r_norm": [0.15823179483413696, 0.15393343567848206, 0.1530448943376541]}, {"step": 6500, "epoch": 16, "train_loss": 0.00038692649104632437, "cos_r_negg": [0.5694202184677124, 0.6485642194747925, 0.681617021560669], "cos_innovation_negg": [0.5694202184677124, 0.6485642194747925, 0.681617021560669], "cos_apical_negg": [0.5694202184677124, 0.6485642194747925, 0.681617021560669], "cos_Ac_negg": [0.5694202184677124, 0.6485642194747925, 0.681617021560669], "r_norm": [0.15785613656044006, 0.1536150574684143, 0.1527000069618225], "ldr": {"dL_sdil": -0.007859811186790466, "dL_bp": -0.014369204640388489, "ratio": 0.5469899958622878}}, {"step": 6550, "epoch": 16, "train_loss": 0.00013543738168664277, "cos_r_negg": [0.5683775544166565, 0.6481328010559082, 0.6817253828048706], "cos_innovation_negg": [0.5683775544166565, 0.6481328010559082, 0.6817253828048706], "cos_apical_negg": [0.5683775544166565, 0.6481328010559082, 0.6817253828048706], "cos_Ac_negg": [0.5683775544166565, 0.6481328010559082, 0.6817253828048706], "r_norm": [0.1583729088306427, 0.15416066348552704, 0.15327590703964233]}, {"step": 6600, "epoch": 16, "train_loss": 0.000376997166313231, "cos_r_negg": [0.5690847635269165, 0.6480680704116821, 0.681452214717865], "cos_innovation_negg": [0.5690847635269165, 0.6480680704116821, 0.681452214717865], "cos_apical_negg": [0.5690847635269165, 0.6480680704116821, 0.681452214717865], "cos_Ac_negg": [0.5690847635269165, 0.6480680704116821, 0.681452214717865], "r_norm": [0.15812790393829346, 0.15385626256465912, 0.1529945582151413]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9708, "test_loss": 0.15816921405792236}, {"step": 6650, "epoch": 17, "train_loss": 0.0004841027839574963, "cos_r_negg": [0.5651593208312988, 0.6472564935684204, 0.6800351738929749], "cos_innovation_negg": [0.5651593208312988, 0.6472564935684204, 0.6800351738929749], "cos_apical_negg": [0.5651593208312988, 0.6472564935684204, 0.6800351738929749], "cos_Ac_negg": [0.5651593208312988, 0.6472564935684204, 0.6800351738929749], "r_norm": [0.15847666561603546, 0.15416184067726135, 0.1533324420452118]}, {"step": 6700, "epoch": 17, "train_loss": 0.0003166238311678171, "cos_r_negg": [0.5664132833480835, 0.647165060043335, 0.6806843280792236], "cos_innovation_negg": [0.5664132833480835, 0.647165060043335, 0.6806843280792236], "cos_apical_negg": [0.5664132833480835, 0.647165060043335, 0.6806843280792236], "cos_Ac_negg": [0.5664132833480835, 0.647165060043335, 0.6806843280792236], "r_norm": [0.15876327455043793, 0.15448659658432007, 0.15352970361709595]}, {"step": 6750, "epoch": 17, "train_loss": 0.0006473849061876535, "cos_r_negg": [0.5671321153640747, 0.6481791138648987, 0.6805248260498047], "cos_innovation_negg": [0.5671321153640747, 0.6481791138648987, 0.6805248260498047], "cos_apical_negg": [0.5671321153640747, 0.6481791138648987, 0.6805248260498047], "cos_Ac_negg": [0.5671321153640747, 0.6481791138648987, 0.6805248260498047], "r_norm": [0.15865540504455566, 0.15440517663955688, 0.1533987820148468], "ldr": {"dL_sdil": -0.007737509906291962, "dL_bp": -0.014118775725364685, "ratio": 0.5480298049066222}}, {"step": 6800, "epoch": 17, "train_loss": 0.0007088524871505797, "cos_r_negg": [0.5687145590782166, 0.6483460664749146, 0.680865466594696], "cos_innovation_negg": [0.5687145590782166, 0.6483460664749146, 0.680865466594696], "cos_apical_negg": [0.5687145590782166, 0.6483460664749146, 0.680865466594696], "cos_Ac_negg": [0.5687145590782166, 0.6483460664749146, 0.680865466594696], "r_norm": [0.15936779975891113, 0.15507258474826813, 0.15408694744110107]}, {"step": 6850, "epoch": 17, "train_loss": 0.00040957663441076875, "cos_r_negg": [0.5670983791351318, 0.6473619341850281, 0.6810409426689148], "cos_innovation_negg": [0.5670983791351318, 0.6473619341850281, 0.6810409426689148], "cos_apical_negg": [0.5670983791351318, 0.6473619341850281, 0.6810409426689148], "cos_Ac_negg": [0.5670983791351318, 0.6473619341850281, 0.6810409426689148], "r_norm": [0.15931186079978943, 0.15504586696624756, 0.15403980016708374]}, {"step": 6900, "epoch": 17, "train_loss": 0.0005429871962405741, "cos_r_negg": [0.5676727890968323, 0.6475051641464233, 0.680949866771698], "cos_innovation_negg": [0.5676727890968323, 0.6475051641464233, 0.680949866771698], "cos_apical_negg": [0.5676727890968323, 0.6475051641464233, 0.680949866771698], "cos_Ac_negg": [0.5676727890968323, 0.6475051641464233, 0.680949866771698], "r_norm": [0.1572607308626175, 0.15304264426231384, 0.1520705223083496]}, {"step": 6950, "epoch": 17, "train_loss": 0.000991414999589324, "cos_r_negg": [0.5668578147888184, 0.6478276252746582, 0.6803035736083984], "cos_innovation_negg": [0.5668578147888184, 0.6478276252746582, 0.6803035736083984], "cos_apical_negg": [0.5668578147888184, 0.6478276252746582, 0.6803035736083984], "cos_Ac_negg": [0.5668578147888184, 0.6478276252746582, 0.6803035736083984], "r_norm": [0.15826283395290375, 0.15402379631996155, 0.15312185883522034]}, {"step": 7000, "epoch": 17, "train_loss": 0.0003208107082173228, "cos_r_negg": [0.566586971282959, 0.6483029127120972, 0.6799967288970947], "cos_innovation_negg": [0.566586971282959, 0.6483029127120972, 0.6799967288970947], "cos_apical_negg": [0.566586971282959, 0.6483029127120972, 0.6799967288970947], "cos_Ac_negg": [0.566586971282959, 0.6483029127120972, 0.6799967288970947], "r_norm": [0.15870504081249237, 0.15444651246070862, 0.1535559743642807], "ldr": {"dL_sdil": -0.00795724242925644, "dL_bp": -0.014492139220237732, "ratio": 0.5490730049118246}}, {"epoch_end": 17, "step": 7038, "test_acc": 0.971, "test_loss": 0.16013721351623536}, {"step": 7050, "epoch": 18, "train_loss": 0.0004086213302798569, "cos_r_negg": [0.5696477293968201, 0.6488428115844727, 0.6800650358200073], "cos_innovation_negg": [0.5696477293968201, 0.6488428115844727, 0.6800650358200073], "cos_apical_negg": [0.5696477293968201, 0.6488428115844727, 0.6800650358200073], "cos_Ac_negg": [0.5696477293968201, 0.6488428115844727, 0.6800650358200073], "r_norm": [0.16048260033130646, 0.15618984401226044, 0.1553167700767517]}, {"step": 7100, "epoch": 18, "train_loss": 0.00028305756859481335, "cos_r_negg": [0.5681222677230835, 0.6485835313796997, 0.6805644631385803], "cos_innovation_negg": [0.5681222677230835, 0.6485835313796997, 0.6805644631385803], "cos_apical_negg": [0.5681222677230835, 0.6485835313796997, 0.6805644631385803], "cos_Ac_negg": [0.5681222677230835, 0.6485835313796997, 0.6805644631385803], "r_norm": [0.16036057472229004, 0.15601018071174622, 0.1551886796951294]}, {"step": 7150, "epoch": 18, "train_loss": 9.6554693300277e-05, "cos_r_negg": [0.568406343460083, 0.6489261388778687, 0.6806586980819702], "cos_innovation_negg": [0.568406343460083, 0.6489261388778687, 0.6806586980819702], "cos_apical_negg": [0.568406343460083, 0.6489261388778687, 0.6806586980819702], "cos_Ac_negg": [0.568406343460083, 0.6489261388778687, 0.6806586980819702], "r_norm": [0.15843012928962708, 0.15420697629451752, 0.15331566333770752]}, {"step": 7200, "epoch": 18, "train_loss": 0.0003479020087979734, "cos_r_negg": [0.5681991577148438, 0.64857417345047, 0.6810015439987183], "cos_innovation_negg": [0.5681991577148438, 0.64857417345047, 0.6810015439987183], "cos_apical_negg": [0.5681991577148438, 0.64857417345047, 0.6810015439987183], "cos_Ac_negg": [0.5681991577148438, 0.64857417345047, 0.6810015439987183], "r_norm": [0.15806657075881958, 0.15387684106826782, 0.15296761691570282]}, {"step": 7250, "epoch": 18, "train_loss": 0.0003669528232421726, "cos_r_negg": [0.5694073438644409, 0.648737907409668, 0.6807727813720703], "cos_innovation_negg": [0.5694073438644409, 0.648737907409668, 0.6807727813720703], "cos_apical_negg": [0.5694073438644409, 0.648737907409668, 0.6807727813720703], "cos_Ac_negg": [0.5694073438644409, 0.648737907409668, 0.6807727813720703], "r_norm": [0.15818016231060028, 0.15396854281425476, 0.15300069749355316], "ldr": {"dL_sdil": -0.007998645305633545, "dL_bp": -0.014995723962783813, "ratio": 0.5333950748549704}}, {"step": 7300, "epoch": 18, "train_loss": 0.0003396801184862852, "cos_r_negg": [0.5685341954231262, 0.6481294631958008, 0.6804102659225464], "cos_innovation_negg": [0.5685341954231262, 0.6481294631958008, 0.6804102659225464], "cos_apical_negg": [0.5685341954231262, 0.6481294631958008, 0.6804102659225464], "cos_Ac_negg": [0.5685341954231262, 0.6481294631958008, 0.6804102659225464], "r_norm": [0.15833894908428192, 0.1541498452425003, 0.15319129824638367]}, {"step": 7350, "epoch": 18, "train_loss": 0.001183387590572238, "cos_r_negg": [0.5686646699905396, 0.6485124230384827, 0.6808387637138367], "cos_innovation_negg": [0.5686646699905396, 0.6485124230384827, 0.6808387637138367], "cos_apical_negg": [0.5686646699905396, 0.6485124230384827, 0.6808387637138367], "cos_Ac_negg": [0.5686646699905396, 0.6485124230384827, 0.6808387637138367], "r_norm": [0.1595434546470642, 0.15531444549560547, 0.15434080362319946]}, {"step": 7400, "epoch": 18, "train_loss": 0.00015149095270317048, "cos_r_negg": [0.5681197643280029, 0.6482692956924438, 0.6805227994918823], "cos_innovation_negg": [0.5681197643280029, 0.6482692956924438, 0.6805227994918823], "cos_apical_negg": [0.5681197643280029, 0.6482692956924438, 0.6805227994918823], "cos_Ac_negg": [0.5681197643280029, 0.6482692956924438, 0.6805227994918823], "r_norm": [0.15895983576774597, 0.154800683259964, 0.15384916961193085]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9712, "test_loss": 0.16168081607818605}, {"step": 7450, "epoch": 19, "train_loss": 5.6864559155656025e-05, "cos_r_negg": [0.5700148940086365, 0.6480550765991211, 0.6811525225639343], "cos_innovation_negg": [0.5700148940086365, 0.6480550765991211, 0.6811525225639343], "cos_apical_negg": [0.5700148940086365, 0.6480550765991211, 0.6811525225639343], "cos_Ac_negg": [0.5700148940086365, 0.6480550765991211, 0.6811525225639343], "r_norm": [0.15917226672172546, 0.15490522980690002, 0.15400728583335876]}, {"step": 7500, "epoch": 19, "train_loss": 0.00022419965534936637, "cos_r_negg": [0.5664976835250854, 0.6474589109420776, 0.6809322834014893], "cos_innovation_negg": [0.5664976835250854, 0.6474589109420776, 0.6809322834014893], "cos_apical_negg": [0.5664976835250854, 0.6474589109420776, 0.6809322834014893], "cos_Ac_negg": [0.5664976835250854, 0.6474589109420776, 0.6809322834014893], "r_norm": [0.1596352607011795, 0.1553056836128235, 0.15447358787059784], "ldr": {"dL_sdil": -0.008223295211791992, "dL_bp": -0.015250787138938904, "ratio": 0.5392046414965661}}, {"step": 7550, "epoch": 19, "train_loss": 0.0002151928492821753, "cos_r_negg": [0.5678735971450806, 0.6470168828964233, 0.6816766858100891], "cos_innovation_negg": [0.5678735971450806, 0.6470168828964233, 0.6816766858100891], "cos_apical_negg": [0.5678735971450806, 0.6470168828964233, 0.6816766858100891], "cos_Ac_negg": [0.5678735971450806, 0.6470168828964233, 0.6816766858100891], "r_norm": [0.1595972180366516, 0.15530681610107422, 0.1543876975774765]}, {"step": 7600, "epoch": 19, "train_loss": 0.00016729974595364183, "cos_r_negg": [0.5694859027862549, 0.6479520797729492, 0.681437075138092], "cos_innovation_negg": [0.5694859027862549, 0.6479520797729492, 0.681437075138092], "cos_apical_negg": [0.5694859027862549, 0.6479520797729492, 0.681437075138092], "cos_Ac_negg": [0.5694859027862549, 0.6479520797729492, 0.681437075138092], "r_norm": [0.15946857631206512, 0.1551889181137085, 0.1543211042881012]}, {"step": 7650, "epoch": 19, "train_loss": 0.0002651689574122429, "cos_r_negg": [0.5680376291275024, 0.6478772163391113, 0.6819467544555664], "cos_innovation_negg": [0.5680376291275024, 0.6478772163391113, 0.6819467544555664], "cos_apical_negg": [0.5680376291275024, 0.6478772163391113, 0.6819467544555664], "cos_Ac_negg": [0.5680376291275024, 0.6478772163391113, 0.6819467544555664], "r_norm": [0.15930406749248505, 0.15504589676856995, 0.15419191122055054]}, {"step": 7700, "epoch": 19, "train_loss": 0.000597011239733547, "cos_r_negg": [0.5686484575271606, 0.6488025784492493, 0.6817859411239624], "cos_innovation_negg": [0.5686484575271606, 0.6488025784492493, 0.6817859411239624], "cos_apical_negg": [0.5686484575271606, 0.6488025784492493, 0.6817859411239624], "cos_Ac_negg": [0.5686484575271606, 0.6488025784492493, 0.6817859411239624], "r_norm": [0.1607024073600769, 0.15636765956878662, 0.1554795205593109]}, {"step": 7750, "epoch": 19, "train_loss": 0.0005972809158265591, "cos_r_negg": [0.5692601799964905, 0.6475745439529419, 0.6824889183044434], "cos_innovation_negg": [0.5692601799964905, 0.6475745439529419, 0.6824889183044434], "cos_apical_negg": [0.5692601799964905, 0.6475745439529419, 0.6824889183044434], "cos_Ac_negg": [0.5692601799964905, 0.6475745439529419, 0.6824889183044434], "r_norm": [0.1594679355621338, 0.1552254855632782, 0.154302716255188], "ldr": {"dL_sdil": -0.00825953483581543, "dL_bp": -0.015581488609313965, "ratio": 0.5300863763991217}}, {"step": 7800, "epoch": 19, "train_loss": 0.00022205314598977566, "cos_r_negg": [0.5694465637207031, 0.6489244699478149, 0.6826184988021851], "cos_innovation_negg": [0.5694465637207031, 0.6489244699478149, 0.6826184988021851], "cos_apical_negg": [0.5694465637207031, 0.6489244699478149, 0.6826184988021851], "cos_Ac_negg": [0.5694465637207031, 0.6489244699478149, 0.6826184988021851], "r_norm": [0.15888230502605438, 0.15462139248847961, 0.15371689200401306]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9712, "test_loss": 0.1633418285369873}, {"step": 7850, "epoch": 20, "train_loss": 0.00046482455218210816, "cos_r_negg": [0.5689709186553955, 0.6480838060379028, 0.6829698085784912], "cos_innovation_negg": [0.5689709186553955, 0.6480838060379028, 0.6829698085784912], "cos_apical_negg": [0.5689709186553955, 0.6480838060379028, 0.6829698085784912], "cos_Ac_negg": [0.5689709186553955, 0.6480838060379028, 0.6829698085784912], "r_norm": [0.15783169865608215, 0.15364302694797516, 0.15275171399116516]}, {"step": 7900, "epoch": 20, "train_loss": 0.0001319045841228217, "cos_r_negg": [0.5696851015090942, 0.6475779414176941, 0.6836149096488953], "cos_innovation_negg": [0.5696851015090942, 0.6475779414176941, 0.6836149096488953], "cos_apical_negg": [0.5696851015090942, 0.6475779414176941, 0.6836149096488953], "cos_Ac_negg": [0.5696851015090942, 0.6475779414176941, 0.6836149096488953], "r_norm": [0.15763188898563385, 0.15345078706741333, 0.15257254242897034]}, {"step": 7950, "epoch": 20, "train_loss": 0.00036133427056483924, "cos_r_negg": [0.5694398880004883, 0.6471772193908691, 0.6827254295349121], "cos_innovation_negg": [0.5694398880004883, 0.6471772193908691, 0.6827254295349121], "cos_apical_negg": [0.5694398880004883, 0.6471772193908691, 0.6827254295349121], "cos_Ac_negg": [0.5694398880004883, 0.6471772193908691, 0.6827254295349121], "r_norm": [0.15701299905776978, 0.1528392881155014, 0.1519688367843628]}, {"step": 8000, "epoch": 20, "train_loss": 0.000248712400207296, "cos_r_negg": [0.5689470171928406, 0.6470829844474792, 0.6831272840499878], "cos_innovation_negg": [0.5689470171928406, 0.6470829844474792, 0.6831272840499878], "cos_apical_negg": [0.5689470171928406, 0.6470829844474792, 0.6831272840499878], "cos_Ac_negg": [0.5689470171928406, 0.6470829844474792, 0.6831272840499878], "r_norm": [0.15693330764770508, 0.1528104990720749, 0.1519172191619873], "ldr": {"dL_sdil": -0.008211091160774231, "dL_bp": -0.015586629509925842, "ratio": 0.5268035116601227}}, {"step": 8050, "epoch": 20, "train_loss": 0.0010373888071626425, "cos_r_negg": [0.5678057074546814, 0.647324800491333, 0.6839549541473389], "cos_innovation_negg": [0.5678057074546814, 0.647324800491333, 0.6839549541473389], "cos_apical_negg": [0.5678057074546814, 0.647324800491333, 0.6839549541473389], "cos_Ac_negg": [0.5678057074546814, 0.647324800491333, 0.6839549541473389], "r_norm": [0.15800955891609192, 0.1538015604019165, 0.15291407704353333]}, {"step": 8100, "epoch": 20, "train_loss": 0.00020829853019677103, "cos_r_negg": [0.5668814182281494, 0.6469424962997437, 0.6826354265213013], "cos_innovation_negg": [0.5668814182281494, 0.6469424962997437, 0.6826354265213013], "cos_apical_negg": [0.5668814182281494, 0.6469424962997437, 0.6826354265213013], "cos_Ac_negg": [0.5668814182281494, 0.6469424962997437, 0.6826354265213013], "r_norm": [0.15864670276641846, 0.15444360673427582, 0.15348567068576813]}, {"step": 8150, "epoch": 20, "train_loss": 0.0003224837710149586, "cos_r_negg": [0.5677663087844849, 0.6470282077789307, 0.6833655834197998], "cos_innovation_negg": [0.5677663087844849, 0.6470282077789307, 0.6833655834197998], "cos_apical_negg": [0.5677663087844849, 0.6470282077789307, 0.6833655834197998], "cos_Ac_negg": [0.5677663087844849, 0.6470282077789307, 0.6833655834197998], "r_norm": [0.15953882038593292, 0.15530505776405334, 0.1544223427772522]}, {"step": 8200, "epoch": 20, "train_loss": 0.00047988706501200795, "cos_r_negg": [0.5670744776725769, 0.6455212831497192, 0.683095395565033], "cos_innovation_negg": [0.5670744776725769, 0.6455212831497192, 0.683095395565033], "cos_apical_negg": [0.5670744776725769, 0.6455212831497192, 0.683095395565033], "cos_Ac_negg": [0.5670744776725769, 0.6455212831497192, 0.683095395565033], "r_norm": [0.15852190554141998, 0.1543133556842804, 0.15345066785812378]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.971, "test_loss": 0.16425190601348877}, {"step": 8250, "epoch": 21, "train_loss": 0.0005111042410135269, "cos_r_negg": [0.5675830841064453, 0.6451117992401123, 0.6833163499832153], "cos_innovation_negg": [0.5675830841064453, 0.6451117992401123, 0.6833163499832153], "cos_apical_negg": [0.5675830841064453, 0.6451117992401123, 0.6833163499832153], "cos_Ac_negg": [0.5675830841064453, 0.6451117992401123, 0.6833163499832153], "r_norm": [0.15747708082199097, 0.15331745147705078, 0.15248464047908783], "ldr": {"dL_sdil": -0.00830041617155075, "dL_bp": -0.015750542283058167, "ratio": 0.5269924058728421}}, {"step": 8300, "epoch": 21, "train_loss": 0.0011355745373293757, "cos_r_negg": [0.5666235089302063, 0.6453575491905212, 0.6824674606323242], "cos_innovation_negg": [0.5666235089302063, 0.6453575491905212, 0.6824674606323242], "cos_apical_negg": [0.5666235089302063, 0.6453575491905212, 0.6824674606323242], "cos_Ac_negg": [0.5666235089302063, 0.6453575491905212, 0.6824674606323242], "r_norm": [0.1571134328842163, 0.15294715762138367, 0.1520799696445465]}, {"step": 8350, "epoch": 21, "train_loss": 0.0005579860298894346, "cos_r_negg": [0.5672865509986877, 0.6450773477554321, 0.6832593679428101], "cos_innovation_negg": [0.5672865509986877, 0.6450773477554321, 0.6832593679428101], "cos_apical_negg": [0.5672865509986877, 0.6450773477554321, 0.6832593679428101], "cos_Ac_negg": [0.5672865509986877, 0.6450773477554321, 0.6832593679428101], "r_norm": [0.15689420700073242, 0.15275223553180695, 0.151885524392128]}, {"step": 8400, "epoch": 21, "train_loss": 0.00035051253507845104, "cos_r_negg": [0.5668098330497742, 0.6453763842582703, 0.6834558248519897], "cos_innovation_negg": [0.5668098330497742, 0.6453763842582703, 0.6834558248519897], "cos_apical_negg": [0.5668098330497742, 0.6453763842582703, 0.6834558248519897], "cos_Ac_negg": [0.5668098330497742, 0.6453763842582703, 0.6834558248519897], "r_norm": [0.15707769989967346, 0.15293119847774506, 0.1520438939332962]}, {"step": 8450, "epoch": 21, "train_loss": 0.0003930824459530413, "cos_r_negg": [0.5667535066604614, 0.6447712182998657, 0.6836518049240112], "cos_innovation_negg": [0.5667535066604614, 0.6447712182998657, 0.6836518049240112], "cos_apical_negg": [0.5667535066604614, 0.6447712182998657, 0.6836518049240112], "cos_Ac_negg": [0.5667535066604614, 0.6447712182998657, 0.6836518049240112], "r_norm": [0.15706190466880798, 0.15288114547729492, 0.15205591917037964]}, {"step": 8500, "epoch": 21, "train_loss": 0.0003577670722734183, "cos_r_negg": [0.5683736801147461, 0.6448572278022766, 0.683910608291626], "cos_innovation_negg": [0.5683736801147461, 0.6448572278022766, 0.683910608291626], "cos_apical_negg": [0.5683736801147461, 0.6448572278022766, 0.683910608291626], "cos_Ac_negg": [0.5683736801147461, 0.6448572278022766, 0.683910608291626], "r_norm": [0.15534566342830658, 0.1512913852930069, 0.15040701627731323], "ldr": {"dL_sdil": -0.008097045123577118, "dL_bp": -0.015407361090183258, "ratio": 0.5255309508346708}}, {"step": 8550, "epoch": 21, "train_loss": 0.0002715772425290197, "cos_r_negg": [0.5686624050140381, 0.6446493864059448, 0.6843215227127075], "cos_innovation_negg": [0.5686624050140381, 0.6446493864059448, 0.6843215227127075], "cos_apical_negg": [0.5686624050140381, 0.6446493864059448, 0.6843215227127075], "cos_Ac_negg": [0.5686624050140381, 0.6446493864059448, 0.6843215227127075], "r_norm": [0.15796007215976715, 0.15375757217407227, 0.15287992358207703]}, {"step": 8600, "epoch": 21, "train_loss": 0.0007725955219939351, "cos_r_negg": [0.5681837201118469, 0.6446834802627563, 0.6843181848526001], "cos_innovation_negg": [0.5681837201118469, 0.6446834802627563, 0.6843181848526001], "cos_apical_negg": [0.5681837201118469, 0.6446834802627563, 0.6843181848526001], "cos_Ac_negg": [0.5681837201118469, 0.6446834802627563, 0.6843181848526001], "r_norm": [0.1584068387746811, 0.1542365550994873, 0.1533406376838684]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9711, "test_loss": 0.1658396402359009}, {"step": 8650, "epoch": 22, "train_loss": 0.00012090055679436773, "cos_r_negg": [0.5683157444000244, 0.644891619682312, 0.6844276189804077], "cos_innovation_negg": [0.5683157444000244, 0.644891619682312, 0.6844276189804077], "cos_apical_negg": [0.5683157444000244, 0.644891619682312, 0.6844276189804077], "cos_Ac_negg": [0.5683157444000244, 0.644891619682312, 0.6844276189804077], "r_norm": [0.15829548239707947, 0.15415874123573303, 0.15331429243087769]}, {"step": 8700, "epoch": 22, "train_loss": 0.00017631097580306232, "cos_r_negg": [0.5682955980300903, 0.6448882818222046, 0.6846705675125122], "cos_innovation_negg": [0.5682955980300903, 0.6448882818222046, 0.6846705675125122], "cos_apical_negg": [0.5682955980300903, 0.6448882818222046, 0.6846705675125122], "cos_Ac_negg": [0.5682955980300903, 0.6448882818222046, 0.6846705675125122], "r_norm": [0.15818250179290771, 0.1540803611278534, 0.15322822332382202]}, {"step": 8750, "epoch": 22, "train_loss": 0.0002886674483306706, "cos_r_negg": [0.568666934967041, 0.6453714966773987, 0.6850155591964722], "cos_innovation_negg": [0.568666934967041, 0.6453714966773987, 0.6850155591964722], "cos_apical_negg": [0.568666934967041, 0.6453714966773987, 0.6850155591964722], "cos_Ac_negg": [0.568666934967041, 0.6453714966773987, 0.6850155591964722], "r_norm": [0.1578974425792694, 0.15380562841892242, 0.1529954969882965], "ldr": {"dL_sdil": -0.008395299315452576, "dL_bp": -0.016024023294448853, "ratio": 0.5239195650920534}}, {"step": 8800, "epoch": 22, "train_loss": 0.00021790500613860786, "cos_r_negg": [0.5693921446800232, 0.6451363563537598, 0.6852111220359802], "cos_innovation_negg": [0.5693921446800232, 0.6451363563537598, 0.6852111220359802], "cos_apical_negg": [0.5693921446800232, 0.6451363563537598, 0.6852111220359802], "cos_Ac_negg": [0.5693921446800232, 0.6451363563537598, 0.6852111220359802], "r_norm": [0.15810224413871765, 0.15395572781562805, 0.15311485528945923]}, {"step": 8850, "epoch": 22, "train_loss": 0.00019700219854712486, "cos_r_negg": [0.5690909624099731, 0.6447280645370483, 0.6851131916046143], "cos_innovation_negg": [0.5690909624099731, 0.6447280645370483, 0.6851131916046143], "cos_apical_negg": [0.5690909624099731, 0.6447280645370483, 0.6851131916046143], "cos_Ac_negg": [0.5690909624099731, 0.6447280645370483, 0.6851131916046143], "r_norm": [0.1586340367794037, 0.1544063687324524, 0.15362460911273956]}, {"step": 8900, "epoch": 22, "train_loss": 0.0006105678621679544, "cos_r_negg": [0.568546712398529, 0.6446664333343506, 0.6853318214416504], "cos_innovation_negg": [0.568546712398529, 0.6446664333343506, 0.6853318214416504], "cos_apical_negg": [0.568546712398529, 0.6446664333343506, 0.6853318214416504], "cos_Ac_negg": [0.568546712398529, 0.6446664333343506, 0.6853318214416504], "r_norm": [0.15937352180480957, 0.15514428913593292, 0.15431244671344757]}, {"step": 8950, "epoch": 22, "train_loss": 0.0004493519081734121, "cos_r_negg": [0.5677918791770935, 0.6445760130882263, 0.6857108473777771], "cos_innovation_negg": [0.5677918791770935, 0.6445760130882263, 0.6857108473777771], "cos_apical_negg": [0.5677918791770935, 0.6445760130882263, 0.6857108473777771], "cos_Ac_negg": [0.5677918791770935, 0.6445760130882263, 0.6857108473777771], "r_norm": [0.15932457149028778, 0.15512004494667053, 0.15423759818077087]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9709, "test_loss": 0.16710832691192626}, {"step": 9000, "epoch": 23, "train_loss": 0.00010449669207446277, "cos_r_negg": [0.5682867765426636, 0.6443584561347961, 0.685322642326355], "cos_innovation_negg": [0.5682867765426636, 0.6443584561347961, 0.685322642326355], "cos_apical_negg": [0.5682867765426636, 0.6443584561347961, 0.685322642326355], "cos_Ac_negg": [0.5682867765426636, 0.6443584561347961, 0.685322642326355], "r_norm": [0.15765559673309326, 0.15348368883132935, 0.1526414304971695], "ldr": {"dL_sdil": -0.008456751704216003, "dL_bp": -0.01632910966873169, "ratio": 0.5178942315764883}}, {"step": 9050, "epoch": 23, "train_loss": 0.0004196309600956738, "cos_r_negg": [0.568623423576355, 0.6445470452308655, 0.6861435174942017], "cos_innovation_negg": [0.568623423576355, 0.6445470452308655, 0.6861435174942017], "cos_apical_negg": [0.568623423576355, 0.6445470452308655, 0.6861435174942017], "cos_Ac_negg": [0.568623423576355, 0.6445470452308655, 0.6861435174942017], "r_norm": [0.15835539996623993, 0.15417641401290894, 0.15332432091236115]}, {"step": 9100, "epoch": 23, "train_loss": 0.00015397371316794306, "cos_r_negg": [0.569488525390625, 0.6447100639343262, 0.6868489980697632], "cos_innovation_negg": [0.569488525390625, 0.6447100639343262, 0.6868489980697632], "cos_apical_negg": [0.569488525390625, 0.6447100639343262, 0.6868489980697632], "cos_Ac_negg": [0.569488525390625, 0.6447100639343262, 0.6868489980697632], "r_norm": [0.15642431378364563, 0.1523085981607437, 0.1514337807893753]}, {"step": 9150, "epoch": 23, "train_loss": 0.00021238671615719795, "cos_r_negg": [0.5682767033576965, 0.6445674300193787, 0.6864808797836304], "cos_innovation_negg": [0.5682767033576965, 0.6445674300193787, 0.6864808797836304], "cos_apical_negg": [0.5682767033576965, 0.6445674300193787, 0.6864808797836304], "cos_Ac_negg": [0.5682767033576965, 0.6445674300193787, 0.6864808797836304], "r_norm": [0.15639150142669678, 0.1523202359676361, 0.15143880248069763]}, {"step": 9200, "epoch": 23, "train_loss": 0.0003630518913269043, "cos_r_negg": [0.5687063932418823, 0.6440786123275757, 0.6864020824432373], "cos_innovation_negg": [0.5687063932418823, 0.6440786123275757, 0.6864020824432373], "cos_apical_negg": [0.5687063932418823, 0.6440786123275757, 0.6864020824432373], "cos_Ac_negg": [0.5687063932418823, 0.6440786123275757, 0.6864020824432373], "r_norm": [0.15650585293769836, 0.15241140127182007, 0.15156108140945435]}, {"step": 9250, "epoch": 23, "train_loss": 0.00016443779168184847, "cos_r_negg": [0.5700812339782715, 0.64438796043396, 0.6869990825653076], "cos_innovation_negg": [0.5700812339782715, 0.64438796043396, 0.6869990825653076], "cos_apical_negg": [0.5700812339782715, 0.64438796043396, 0.6869990825653076], "cos_Ac_negg": [0.5700812339782715, 0.64438796043396, 0.6869990825653076], "r_norm": [0.15573111176490784, 0.1516374796628952, 0.1507754921913147], "ldr": {"dL_sdil": -0.008447304368019104, "dL_bp": -0.016437843441963196, "ratio": 0.5138937110481586}}, {"step": 9300, "epoch": 23, "train_loss": 0.00012251742009539157, "cos_r_negg": [0.5682557821273804, 0.6446056365966797, 0.6867387294769287], "cos_innovation_negg": [0.5682557821273804, 0.6446056365966797, 0.6867387294769287], "cos_apical_negg": [0.5682557821273804, 0.6446056365966797, 0.6867387294769287], "cos_Ac_negg": [0.5682557821273804, 0.6446056365966797, 0.6867387294769287], "r_norm": [0.15695522725582123, 0.15281346440315247, 0.15199807286262512]}, {"step": 9350, "epoch": 23, "train_loss": 0.000272915989626199, "cos_r_negg": [0.5679590702056885, 0.6448975801467896, 0.6872338056564331], "cos_innovation_negg": [0.5679590702056885, 0.6448975801467896, 0.6872338056564331], "cos_apical_negg": [0.5679590702056885, 0.6448975801467896, 0.6872338056564331], "cos_Ac_negg": [0.5679590702056885, 0.6448975801467896, 0.6872338056564331], "r_norm": [0.15664231777191162, 0.15249375998973846, 0.1517341583967209]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9707, "test_loss": 0.16845075187683106}, {"step": 9400, "epoch": 24, "train_loss": 0.0003718377847690135, "cos_r_negg": [0.5691211819648743, 0.6439048051834106, 0.6876119375228882], "cos_innovation_negg": [0.5691211819648743, 0.6439048051834106, 0.6876119375228882], "cos_apical_negg": [0.5691211819648743, 0.6439048051834106, 0.6876119375228882], "cos_Ac_negg": [0.5691211819648743, 0.6439048051834106, 0.6876119375228882], "r_norm": [0.1583208441734314, 0.1541619598865509, 0.15330779552459717]}, {"step": 9450, "epoch": 24, "train_loss": 6.345242582028732e-05, "cos_r_negg": [0.5693155527114868, 0.6445938944816589, 0.687631368637085], "cos_innovation_negg": [0.5693155527114868, 0.6445938944816589, 0.687631368637085], "cos_apical_negg": [0.5693155527114868, 0.6445938944816589, 0.687631368637085], "cos_Ac_negg": [0.5693155527114868, 0.6445938944816589, 0.687631368637085], "r_norm": [0.15788623690605164, 0.15374627709388733, 0.1529136598110199]}, {"step": 9500, "epoch": 24, "train_loss": 0.00022349027858581394, "cos_r_negg": [0.5683730244636536, 0.6445119380950928, 0.6878803968429565], "cos_innovation_negg": [0.5683730244636536, 0.6445119380950928, 0.6878803968429565], "cos_apical_negg": [0.5683730244636536, 0.6445119380950928, 0.6878803968429565], "cos_Ac_negg": [0.5683730244636536, 0.6445119380950928, 0.6878803968429565], "r_norm": [0.15718498826026917, 0.1530420482158661, 0.1522001326084137], "ldr": {"dL_sdil": -0.008492045104503632, "dL_bp": -0.01652204990386963, "ratio": 0.5139825356970209}}, {"step": 9550, "epoch": 24, "train_loss": 0.00013137566566001624, "cos_r_negg": [0.5689926147460938, 0.644115149974823, 0.6879661083221436], "cos_innovation_negg": [0.5689926147460938, 0.644115149974823, 0.6879661083221436], "cos_apical_negg": [0.5689926147460938, 0.644115149974823, 0.6879661083221436], "cos_Ac_negg": [0.5689926147460938, 0.644115149974823, 0.6879661083221436], "r_norm": [0.15749099850654602, 0.15332409739494324, 0.152482271194458]}, {"step": 9600, "epoch": 24, "train_loss": 0.0002159822906833142, "cos_r_negg": [0.567863941192627, 0.6443837285041809, 0.6879081726074219], "cos_innovation_negg": [0.567863941192627, 0.6443837285041809, 0.6879081726074219], "cos_apical_negg": [0.567863941192627, 0.6443837285041809, 0.6879081726074219], "cos_Ac_negg": [0.567863941192627, 0.6443837285041809, 0.6879081726074219], "r_norm": [0.15738502144813538, 0.15322327613830566, 0.1523977518081665]}, {"step": 9650, "epoch": 24, "train_loss": 0.00032132401247508824, "cos_r_negg": [0.5677769184112549, 0.6441502571105957, 0.6878767013549805], "cos_innovation_negg": [0.5677769184112549, 0.6441502571105957, 0.6878767013549805], "cos_apical_negg": [0.5677769184112549, 0.6441502571105957, 0.6878767013549805], "cos_Ac_negg": [0.5677769184112549, 0.6441502571105957, 0.6878767013549805], "r_norm": [0.15696373581886292, 0.152853325009346, 0.1520533561706543]}, {"step": 9700, "epoch": 24, "train_loss": 0.00014817439659964293, "cos_r_negg": [0.5664092302322388, 0.6439014673233032, 0.687190055847168], "cos_innovation_negg": [0.5664092302322388, 0.6439014673233032, 0.687190055847168], "cos_apical_negg": [0.5664092302322388, 0.6439014673233032, 0.687190055847168], "cos_Ac_negg": [0.5664092302322388, 0.6439014673233032, 0.687190055847168], "r_norm": [0.15771982073783875, 0.15363836288452148, 0.15275359153747559]}, {"step": 9750, "epoch": 24, "train_loss": 0.00041539312223903835, "cos_r_negg": [0.5669523477554321, 0.6444856524467468, 0.688122034072876], "cos_innovation_negg": [0.5669523477554321, 0.6444856524467468, 0.688122034072876], "cos_apical_negg": [0.5669523477554321, 0.6444856524467468, 0.688122034072876], "cos_Ac_negg": [0.5669523477554321, 0.6444856524467468, 0.688122034072876], "r_norm": [0.15748190879821777, 0.15337103605270386, 0.15254215896129608], "ldr": {"dL_sdil": -0.008642807602882385, "dL_bp": -0.016867443919181824, "ratio": 0.5123958107875313}}, {"epoch_end": 24, "step": 9775, "test_acc": 0.971, "test_loss": 0.16931297550201416}], "final": {"test_acc": 0.971, "test_loss": 0.16931297550201416, "wall_s": 29.162970781326294, "cos_r_negg": [0.5670695304870605, 0.6444252729415894, 0.6881203651428223], "cos_innovation_negg": [0.5670695304870605, 0.6444252729415894, 0.6881203651428223], "cos_apical_negg": [0.5670695304870605, 0.6444252729415894, 0.6881203651428223], "cos_Ac_negg": [0.5670695304870605, 0.6444252729415894, 0.6881203651428223]}} \ No newline at end of file diff --git a/results/ep_match_v1_mnist_sdil_w256_d3_s1.json b/results/ep_match_v1_mnist_sdil_w256_d3_s1.json new file mode 100644 index 0000000..9977efd --- /dev/null +++ b/results/ep_match_v1_mnist_sdil_w256_d3_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 25, "batch_size": 128, "train_examples": 50000, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 50, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "ep_match_v1_mnist_sdil_w256_d3_s1", "n_in": 784}, "provenance": {"git_commit": "262c6d9ef3f0461218c1a5e632fc7dd6357e0a20", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3325631618499756, "cos_r_negg": [0.018154017627239227, -0.011148737743496895, -0.005816261284053326], "cos_innovation_negg": [0.018154017627239227, -0.011148737743496895, -0.005816261284053326], "cos_apical_negg": [0.018154017627239227, -0.011148737743496895, -0.005816261284053326], "cos_Ac_negg": [0.018154017627239227, -0.011148737743496895, -0.005816261284053326], "r_norm": [4.733996868133545, 4.796762466430664, 4.7863969802856445], "ldr": {"dL_sdil": -0.04592156410217285, "dL_bp": -0.22312402725219727, "ratio": 0.20581182881872095}}, {"step": 50, "epoch": 0, "train_loss": 0.6989476680755615, "cos_r_negg": [0.4171985387802124, 0.31294262409210205, 0.29292982816696167], "cos_innovation_negg": [0.4171985387802124, 0.31294262409210205, 0.29292982816696167], "cos_apical_negg": [0.4171985387802124, 0.31294262409210205, 0.29292982816696167], "cos_Ac_negg": [0.4171985387802124, 0.31294262409210205, 0.29292982816696167], "r_norm": [1.2128924131393433, 1.216951608657837, 1.2426297664642334]}, {"step": 100, "epoch": 0, "train_loss": 0.30894869565963745, "cos_r_negg": [0.46398890018463135, 0.3279477655887604, 0.3241901397705078], "cos_innovation_negg": [0.46398890018463135, 0.3279477655887604, 0.3241901397705078], "cos_apical_negg": [0.46398890018463135, 0.3279477655887604, 0.3241901397705078], "cos_Ac_negg": [0.46398890018463135, 0.3279477655887604, 0.3241901397705078], "r_norm": [0.9851197004318237, 0.9905135631561279, 1.0092172622680664]}, {"step": 150, "epoch": 0, "train_loss": 0.18788672983646393, "cos_r_negg": [0.45866602659225464, 0.371407687664032, 0.3326675295829773], "cos_innovation_negg": [0.45866602659225464, 0.371407687664032, 0.3326675295829773], "cos_apical_negg": [0.45866602659225464, 0.371407687664032, 0.3326675295829773], "cos_Ac_negg": [0.45866602659225464, 0.371407687664032, 0.3326675295829773], "r_norm": [0.7670449018478394, 0.772146463394165, 0.7833980321884155]}, {"step": 200, "epoch": 0, "train_loss": 0.28179308772087097, "cos_r_negg": [0.4737855792045593, 0.40062761306762695, 0.3733294606208801], "cos_innovation_negg": [0.4737855792045593, 0.40062761306762695, 0.3733294606208801], "cos_apical_negg": [0.4737855792045593, 0.40062761306762695, 0.3733294606208801], "cos_Ac_negg": [0.4737855792045593, 0.40062761306762695, 0.3733294606208801], "r_norm": [0.7265447378158569, 0.7328870892524719, 0.7447304725646973]}, {"step": 250, "epoch": 0, "train_loss": 0.26669591665267944, "cos_r_negg": [0.5002297759056091, 0.4202895164489746, 0.37864184379577637], "cos_innovation_negg": [0.5002297759056091, 0.4202895164489746, 0.37864184379577637], "cos_apical_negg": [0.5002297759056091, 0.4202895164489746, 0.37864184379577637], "cos_Ac_negg": [0.5002297759056091, 0.4202895164489746, 0.37864184379577637], "r_norm": [0.6529134511947632, 0.653572678565979, 0.6668415665626526], "ldr": {"dL_sdil": -0.010019272565841675, "dL_bp": -0.008777543902397156, "ratio": 1.1414665283651162}}, {"step": 300, "epoch": 0, "train_loss": 0.1448812037706375, "cos_r_negg": [0.5132012367248535, 0.429262638092041, 0.370262086391449], "cos_innovation_negg": [0.5132012367248535, 0.429262638092041, 0.370262086391449], "cos_apical_negg": [0.5132012367248535, 0.429262638092041, 0.370262086391449], "cos_Ac_negg": [0.5132012367248535, 0.429262638092041, 0.370262086391449], "r_norm": [0.5609323978424072, 0.5626816749572754, 0.5759875774383545]}, {"step": 350, "epoch": 0, "train_loss": 0.2116699069738388, "cos_r_negg": [0.5159497261047363, 0.450511634349823, 0.37895625829696655], "cos_innovation_negg": [0.5159497261047363, 0.450511634349823, 0.37895625829696655], "cos_apical_negg": [0.5159497261047363, 0.450511634349823, 0.37895625829696655], "cos_Ac_negg": [0.5159497261047363, 0.450511634349823, 0.37895625829696655], "r_norm": [0.5851317644119263, 0.5871607065200806, 0.5960401296615601]}, {"epoch_end": 0, "step": 391, "test_acc": 0.9425, "test_loss": 0.19122634506225586}, {"step": 400, "epoch": 1, "train_loss": 0.12420488893985748, "cos_r_negg": [0.518089771270752, 0.47889840602874756, 0.4130493998527527], "cos_innovation_negg": [0.518089771270752, 0.47889840602874756, 0.4130493998527527], "cos_apical_negg": [0.518089771270752, 0.47889840602874756, 0.4130493998527527], "cos_Ac_negg": [0.518089771270752, 0.47889840602874756, 0.4130493998527527], "r_norm": [0.5363465547561646, 0.5381661653518677, 0.5455495715141296]}, {"step": 450, "epoch": 1, "train_loss": 0.16129380464553833, "cos_r_negg": [0.4891979396343231, 0.4537259042263031, 0.3956911563873291], "cos_innovation_negg": [0.4891979396343231, 0.4537259042263031, 0.3956911563873291], "cos_apical_negg": [0.4891979396343231, 0.4537259042263031, 0.3956911563873291], "cos_Ac_negg": [0.4891979396343231, 0.4537259042263031, 0.3956911563873291], "r_norm": [0.46002060174942017, 0.4591332674026489, 0.4687613546848297]}, {"step": 500, "epoch": 1, "train_loss": 0.1766093671321869, "cos_r_negg": [0.5030666589736938, 0.4638838768005371, 0.40035736560821533], "cos_innovation_negg": [0.5030666589736938, 0.4638838768005371, 0.40035736560821533], "cos_apical_negg": [0.5030666589736938, 0.4638838768005371, 0.40035736560821533], "cos_Ac_negg": [0.5030666589736938, 0.4638838768005371, 0.40035736560821533], "r_norm": [0.4260190725326538, 0.42835742235183716, 0.4325617849826813], "ldr": {"dL_sdil": -0.005232229828834534, "dL_bp": -0.004542216658592224, "ratio": 1.1519111090698537}}, {"step": 550, "epoch": 1, "train_loss": 0.13919229805469513, "cos_r_negg": [0.5142093896865845, 0.4778963327407837, 0.41514936089515686], "cos_innovation_negg": [0.5142093896865845, 0.4778963327407837, 0.41514936089515686], "cos_apical_negg": [0.5142093896865845, 0.4778963327407837, 0.41514936089515686], "cos_Ac_negg": [0.5142093896865845, 0.4778963327407837, 0.41514936089515686], "r_norm": [0.4577493369579315, 0.45764464139938354, 0.46386289596557617]}, {"step": 600, "epoch": 1, "train_loss": 0.10664363205432892, "cos_r_negg": [0.5136488676071167, 0.4854944348335266, 0.42114052176475525], "cos_innovation_negg": [0.5136488676071167, 0.4854944348335266, 0.42114052176475525], "cos_apical_negg": [0.5136488676071167, 0.4854944348335266, 0.42114052176475525], "cos_Ac_negg": [0.5136488676071167, 0.4854944348335266, 0.42114052176475525], "r_norm": [0.45580190420150757, 0.45935192704200745, 0.4583771228790283]}, {"step": 650, "epoch": 1, "train_loss": 0.11938028037548065, "cos_r_negg": [0.5378670692443848, 0.4822646975517273, 0.4304743707180023], "cos_innovation_negg": [0.5378670692443848, 0.4822646975517273, 0.4304743707180023], "cos_apical_negg": [0.5378670692443848, 0.4822646975517273, 0.4304743707180023], "cos_Ac_negg": [0.5378670692443848, 0.4822646975517273, 0.4304743707180023], "r_norm": [0.3810032606124878, 0.380464643239975, 0.3854506313800812]}, {"step": 700, "epoch": 1, "train_loss": 0.12411926686763763, "cos_r_negg": [0.521552324295044, 0.4769900441169739, 0.4258943796157837], "cos_innovation_negg": [0.521552324295044, 0.4769900441169739, 0.4258943796157837], "cos_apical_negg": [0.521552324295044, 0.4769900441169739, 0.4258943796157837], "cos_Ac_negg": [0.521552324295044, 0.4769900441169739, 0.4258943796157837], "r_norm": [0.4334273934364319, 0.4388948082923889, 0.439134418964386]}, {"step": 750, "epoch": 1, "train_loss": 0.19310076534748077, "cos_r_negg": [0.5466012358665466, 0.5047886371612549, 0.4239911437034607], "cos_innovation_negg": [0.5466012358665466, 0.5047886371612549, 0.4239911437034607], "cos_apical_negg": [0.5466012358665466, 0.5047886371612549, 0.4239911437034607], "cos_Ac_negg": [0.5466012358665466, 0.5047886371612549, 0.4239911437034607], "r_norm": [0.4452641010284424, 0.45057982206344604, 0.4524746835231781], "ldr": {"dL_sdil": -0.010885864496231079, "dL_bp": -0.009648069739341736, "ratio": 1.1282945490994964}}, {"epoch_end": 1, "step": 782, "test_acc": 0.9559, "test_loss": 0.14776166839599608}, {"step": 800, "epoch": 2, "train_loss": 0.11163865029811859, "cos_r_negg": [0.5213137269020081, 0.5032415390014648, 0.4327084422111511], "cos_innovation_negg": [0.5213137269020081, 0.5032415390014648, 0.4327084422111511], "cos_apical_negg": [0.5213137269020081, 0.5032415390014648, 0.4327084422111511], "cos_Ac_negg": [0.5213137269020081, 0.5032415390014648, 0.4327084422111511], "r_norm": [0.44324442744255066, 0.4472297430038452, 0.4500628411769867]}, {"step": 850, "epoch": 2, "train_loss": 0.039059530943632126, "cos_r_negg": [0.5520291924476624, 0.5212968587875366, 0.4601536989212036], "cos_innovation_negg": [0.5520291924476624, 0.5212968587875366, 0.4601536989212036], "cos_apical_negg": [0.5520291924476624, 0.5212968587875366, 0.4601536989212036], "cos_Ac_negg": [0.5520291924476624, 0.5212968587875366, 0.4601536989212036], "r_norm": [0.3407784700393677, 0.3428008556365967, 0.343068391084671]}, {"step": 900, "epoch": 2, "train_loss": 0.046419657766819, "cos_r_negg": [0.5510814189910889, 0.5161781907081604, 0.44912809133529663], "cos_innovation_negg": [0.5510814189910889, 0.5161781907081604, 0.44912809133529663], "cos_apical_negg": [0.5510814189910889, 0.5161781907081604, 0.44912809133529663], "cos_Ac_negg": [0.5510814189910889, 0.5161781907081604, 0.44912809133529663], "r_norm": [0.38741910457611084, 0.38852521777153015, 0.3908008337020874]}, {"step": 950, "epoch": 2, "train_loss": 0.10146347433328629, "cos_r_negg": [0.5576192140579224, 0.5226948261260986, 0.4546542167663574], "cos_innovation_negg": [0.5576192140579224, 0.5226948261260986, 0.4546542167663574], "cos_apical_negg": [0.5576192140579224, 0.5226948261260986, 0.4546542167663574], "cos_Ac_negg": [0.5576192140579224, 0.5226948261260986, 0.4546542167663574], "r_norm": [0.34414437413215637, 0.3441411852836609, 0.3469480276107788]}, {"step": 1000, "epoch": 2, "train_loss": 0.1487901210784912, "cos_r_negg": [0.5535523295402527, 0.5312702655792236, 0.45381706953048706], "cos_innovation_negg": [0.5535523295402527, 0.5312702655792236, 0.45381706953048706], "cos_apical_negg": [0.5535523295402527, 0.5312702655792236, 0.45381706953048706], "cos_Ac_negg": [0.5535523295402527, 0.5312702655792236, 0.45381706953048706], "r_norm": [0.3766879141330719, 0.3780704736709595, 0.377485454082489], "ldr": {"dL_sdil": -0.008726179599761963, "dL_bp": -0.008468881249427795, "ratio": 1.0303816221713527}}, {"step": 1050, "epoch": 2, "train_loss": 0.058551136404275894, "cos_r_negg": [0.5609028339385986, 0.533680260181427, 0.4555811285972595], "cos_innovation_negg": [0.5609028339385986, 0.533680260181427, 0.4555811285972595], "cos_apical_negg": [0.5609028339385986, 0.533680260181427, 0.4555811285972595], "cos_Ac_negg": [0.5609028339385986, 0.533680260181427, 0.4555811285972595], "r_norm": [0.38181230425834656, 0.3790220022201538, 0.3856940269470215]}, {"step": 1100, "epoch": 2, "train_loss": 0.17687751352787018, "cos_r_negg": [0.5474849939346313, 0.5372866988182068, 0.4660332202911377], "cos_innovation_negg": [0.5474849939346313, 0.5372866988182068, 0.4660332202911377], "cos_apical_negg": [0.5474849939346313, 0.5372866988182068, 0.4660332202911377], "cos_Ac_negg": [0.5474849939346313, 0.5372866988182068, 0.4660332202911377], "r_norm": [0.30651408433914185, 0.30809253454208374, 0.30758780241012573]}, {"step": 1150, "epoch": 2, "train_loss": 0.1137852594256401, "cos_r_negg": [0.5690146684646606, 0.5531913638114929, 0.4729084372520447], "cos_innovation_negg": [0.5690146684646606, 0.5531913638114929, 0.4729084372520447], "cos_apical_negg": [0.5690146684646606, 0.5531913638114929, 0.4729084372520447], "cos_Ac_negg": [0.5690146684646606, 0.5531913638114929, 0.4729084372520447], "r_norm": [0.2800094783306122, 0.28288063406944275, 0.2845507562160492]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.9607, "test_loss": 0.13978848457336426}, {"step": 1200, "epoch": 3, "train_loss": 0.12797203660011292, "cos_r_negg": [0.5391243696212769, 0.5220585465431213, 0.47794824838638306], "cos_innovation_negg": [0.5391243696212769, 0.5220585465431213, 0.47794824838638306], "cos_apical_negg": [0.5391243696212769, 0.5220585465431213, 0.47794824838638306], "cos_Ac_negg": [0.5391243696212769, 0.5220585465431213, 0.47794824838638306], "r_norm": [0.34349167346954346, 0.34418725967407227, 0.34361711144447327]}, {"step": 1250, "epoch": 3, "train_loss": 0.06349731236696243, "cos_r_negg": [0.5545969009399414, 0.552543044090271, 0.4937087893486023], "cos_innovation_negg": [0.5545969009399414, 0.552543044090271, 0.4937087893486023], "cos_apical_negg": [0.5545969009399414, 0.552543044090271, 0.4937087893486023], "cos_Ac_negg": [0.5545969009399414, 0.552543044090271, 0.4937087893486023], "r_norm": [0.3142988085746765, 0.31554102897644043, 0.31786662340164185], "ldr": {"dL_sdil": -0.008635275065898895, "dL_bp": -0.00844060629606247, "ratio": 1.0230633633424222}}, {"step": 1300, "epoch": 3, "train_loss": 0.03290410339832306, "cos_r_negg": [0.5599110722541809, 0.549652099609375, 0.5057892799377441], "cos_innovation_negg": [0.5599110722541809, 0.549652099609375, 0.5057892799377441], "cos_apical_negg": [0.5599110722541809, 0.549652099609375, 0.5057892799377441], "cos_Ac_negg": [0.5599110722541809, 0.549652099609375, 0.5057892799377441], "r_norm": [0.3252946734428406, 0.3250690996646881, 0.3266938328742981]}, {"step": 1350, "epoch": 3, "train_loss": 0.03429228067398071, "cos_r_negg": [0.5503717660903931, 0.5527141690254211, 0.5055323839187622], "cos_innovation_negg": [0.5503717660903931, 0.5527141690254211, 0.5055323839187622], "cos_apical_negg": [0.5503717660903931, 0.5527141690254211, 0.5055323839187622], "cos_Ac_negg": [0.5503717660903931, 0.5527141690254211, 0.5055323839187622], "r_norm": [0.3082658648490906, 0.3064754605293274, 0.30780020356178284]}, {"step": 1400, "epoch": 3, "train_loss": 0.021261194720864296, "cos_r_negg": [0.5700349807739258, 0.57642662525177, 0.5124076008796692], "cos_innovation_negg": [0.5700349807739258, 0.57642662525177, 0.5124076008796692], "cos_apical_negg": [0.5700349807739258, 0.57642662525177, 0.5124076008796692], "cos_Ac_negg": [0.5700349807739258, 0.57642662525177, 0.5124076008796692], "r_norm": [0.3208509087562561, 0.32245934009552, 0.3249557614326477]}, {"step": 1450, "epoch": 3, "train_loss": 0.03150917589664459, "cos_r_negg": [0.565904438495636, 0.5654979348182678, 0.5234019756317139], "cos_innovation_negg": [0.565904438495636, 0.5654979348182678, 0.5234019756317139], "cos_apical_negg": [0.565904438495636, 0.5654979348182678, 0.5234019756317139], "cos_Ac_negg": [0.565904438495636, 0.5654979348182678, 0.5234019756317139], "r_norm": [0.33029210567474365, 0.32985401153564453, 0.3303276300430298]}, {"step": 1500, "epoch": 3, "train_loss": 0.15419772267341614, "cos_r_negg": [0.5612071752548218, 0.5787191987037659, 0.5163246989250183], "cos_innovation_negg": [0.5612071752548218, 0.5787191987037659, 0.5163246989250183], "cos_apical_negg": [0.5612071752548218, 0.5787191987037659, 0.5163246989250183], "cos_Ac_negg": [0.5612071752548218, 0.5787191987037659, 0.5163246989250183], "r_norm": [0.27825573086738586, 0.2751688063144684, 0.276800274848938], "ldr": {"dL_sdil": -0.006136387586593628, "dL_bp": -0.006221778690814972, "ratio": 0.9862754513677248}}, {"step": 1550, "epoch": 3, "train_loss": 0.0912209302186966, "cos_r_negg": [0.5553269386291504, 0.5711696743965149, 0.5167385339736938], "cos_innovation_negg": [0.5553269386291504, 0.5711696743965149, 0.5167385339736938], "cos_apical_negg": [0.5553269386291504, 0.5711696743965149, 0.5167385339736938], "cos_Ac_negg": [0.5553269386291504, 0.5711696743965149, 0.5167385339736938], "r_norm": [0.2759833335876465, 0.2753640115261078, 0.2756248712539673]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.9639, "test_loss": 0.12990061645507814}, {"step": 1600, "epoch": 4, "train_loss": 0.05743440240621567, "cos_r_negg": [0.5607352256774902, 0.5780284404754639, 0.5336633920669556], "cos_innovation_negg": [0.5607352256774902, 0.5780284404754639, 0.5336633920669556], "cos_apical_negg": [0.5607352256774902, 0.5780284404754639, 0.5336633920669556], "cos_Ac_negg": [0.5607352256774902, 0.5780284404754639, 0.5336633920669556], "r_norm": [0.28030359745025635, 0.27923378348350525, 0.27872714400291443]}, {"step": 1650, "epoch": 4, "train_loss": 0.14237824082374573, "cos_r_negg": [0.5579023361206055, 0.579719066619873, 0.5433968901634216], "cos_innovation_negg": [0.5579023361206055, 0.579719066619873, 0.5433968901634216], "cos_apical_negg": [0.5579023361206055, 0.579719066619873, 0.5433968901634216], "cos_Ac_negg": [0.5579023361206055, 0.579719066619873, 0.5433968901634216], "r_norm": [0.27941441535949707, 0.27573639154434204, 0.27639150619506836]}, {"step": 1700, "epoch": 4, "train_loss": 0.0349678136408329, "cos_r_negg": [0.5651949644088745, 0.5925112962722778, 0.5498974323272705], "cos_innovation_negg": [0.5651949644088745, 0.5925112962722778, 0.5498974323272705], "cos_apical_negg": [0.5651949644088745, 0.5925112962722778, 0.5498974323272705], "cos_Ac_negg": [0.5651949644088745, 0.5925112962722778, 0.5498974323272705], "r_norm": [0.22806960344314575, 0.22662368416786194, 0.2286100536584854]}, {"step": 1750, "epoch": 4, "train_loss": 0.06893368065357208, "cos_r_negg": [0.5756667852401733, 0.5938726663589478, 0.549986720085144], "cos_innovation_negg": [0.5756667852401733, 0.5938726663589478, 0.549986720085144], "cos_apical_negg": [0.5756667852401733, 0.5938726663589478, 0.549986720085144], "cos_Ac_negg": [0.5756667852401733, 0.5938726663589478, 0.549986720085144], "r_norm": [0.2572152614593506, 0.25760746002197266, 0.25760915875434875], "ldr": {"dL_sdil": -0.005817413330078125, "dL_bp": -0.00613471120595932, "ratio": 0.9482782701208544}}, {"step": 1800, "epoch": 4, "train_loss": 0.011489680036902428, "cos_r_negg": [0.5706897974014282, 0.5825215578079224, 0.5430774688720703], "cos_innovation_negg": [0.5706897974014282, 0.5825215578079224, 0.5430774688720703], "cos_apical_negg": [0.5706897974014282, 0.5825215578079224, 0.5430774688720703], "cos_Ac_negg": [0.5706897974014282, 0.5825215578079224, 0.5430774688720703], "r_norm": [0.2798285186290741, 0.2789258658885956, 0.2790088653564453]}, {"step": 1850, "epoch": 4, "train_loss": 0.17075984179973602, "cos_r_negg": [0.5895522832870483, 0.5923226475715637, 0.5501309037208557], "cos_innovation_negg": [0.5895522832870483, 0.5923226475715637, 0.5501309037208557], "cos_apical_negg": [0.5895522832870483, 0.5923226475715637, 0.5501309037208557], "cos_Ac_negg": [0.5895522832870483, 0.5923226475715637, 0.5501309037208557], "r_norm": [0.24603775143623352, 0.24499088525772095, 0.24440163373947144]}, {"step": 1900, "epoch": 4, "train_loss": 0.03888227790594101, "cos_r_negg": [0.604625940322876, 0.608974039554596, 0.5572612285614014], "cos_innovation_negg": [0.604625940322876, 0.608974039554596, 0.5572612285614014], "cos_apical_negg": [0.604625940322876, 0.608974039554596, 0.5572612285614014], "cos_Ac_negg": [0.604625940322876, 0.608974039554596, 0.5572612285614014], "r_norm": [0.263355016708374, 0.2608250677585602, 0.26052603125572205]}, {"step": 1950, "epoch": 4, "train_loss": 0.07593891769647598, "cos_r_negg": [0.5777276754379272, 0.6056712865829468, 0.5574997663497925], "cos_innovation_negg": [0.5777276754379272, 0.6056712865829468, 0.5574997663497925], "cos_apical_negg": [0.5777276754379272, 0.6056712865829468, 0.5574997663497925], "cos_Ac_negg": [0.5777276754379272, 0.6056712865829468, 0.5574997663497925], "r_norm": [0.23587922751903534, 0.23475903272628784, 0.23408223688602448]}, {"epoch_end": 4, "step": 1955, "test_acc": 0.967, "test_loss": 0.12337629852294922}, {"step": 2000, "epoch": 5, "train_loss": 0.023494964465498924, "cos_r_negg": [0.5908063650131226, 0.6100976467132568, 0.5627651214599609], "cos_innovation_negg": [0.5908063650131226, 0.6100976467132568, 0.5627651214599609], "cos_apical_negg": [0.5908063650131226, 0.6100976467132568, 0.5627651214599609], "cos_Ac_negg": [0.5908063650131226, 0.6100976467132568, 0.5627651214599609], "r_norm": [0.22483916580677032, 0.22466060519218445, 0.2251742035150528], "ldr": {"dL_sdil": -0.005233637988567352, "dL_bp": -0.005868658423423767, "ratio": 0.8917946165819027}}, {"step": 2050, "epoch": 5, "train_loss": 0.012155456468462944, "cos_r_negg": [0.5852792263031006, 0.6092922687530518, 0.5668818950653076], "cos_innovation_negg": [0.5852792263031006, 0.6092922687530518, 0.5668818950653076], "cos_apical_negg": [0.5852792263031006, 0.6092922687530518, 0.5668818950653076], "cos_Ac_negg": [0.5852792263031006, 0.6092922687530518, 0.5668818950653076], "r_norm": [0.2536282539367676, 0.25350916385650635, 0.25413376092910767]}, {"step": 2100, "epoch": 5, "train_loss": 0.010819458402693272, "cos_r_negg": [0.6041580438613892, 0.6306443214416504, 0.5843192338943481], "cos_innovation_negg": [0.6041580438613892, 0.6306443214416504, 0.5843192338943481], "cos_apical_negg": [0.6041580438613892, 0.6306443214416504, 0.5843192338943481], "cos_Ac_negg": [0.6041580438613892, 0.6306443214416504, 0.5843192338943481], "r_norm": [0.24017031490802765, 0.24073345959186554, 0.23874536156654358]}, {"step": 2150, "epoch": 5, "train_loss": 0.005749191157519817, "cos_r_negg": [0.5904200077056885, 0.616120457649231, 0.5784893035888672], "cos_innovation_negg": [0.5904200077056885, 0.616120457649231, 0.5784893035888672], "cos_apical_negg": [0.5904200077056885, 0.616120457649231, 0.5784893035888672], "cos_Ac_negg": [0.5904200077056885, 0.616120457649231, 0.5784893035888672], "r_norm": [0.23729190230369568, 0.23729483783245087, 0.23779061436653137]}, {"step": 2200, "epoch": 5, "train_loss": 0.03152646869421005, "cos_r_negg": [0.5832079648971558, 0.6068599224090576, 0.5781339406967163], "cos_innovation_negg": [0.5832079648971558, 0.6068599224090576, 0.5781339406967163], "cos_apical_negg": [0.5832079648971558, 0.6068599224090576, 0.5781339406967163], "cos_Ac_negg": [0.5832079648971558, 0.6068599224090576, 0.5781339406967163], "r_norm": [0.2263680100440979, 0.22405868768692017, 0.22291071712970734]}, {"step": 2250, "epoch": 5, "train_loss": 0.028779540210962296, "cos_r_negg": [0.5948014855384827, 0.6161602735519409, 0.5875178575515747], "cos_innovation_negg": [0.5948014855384827, 0.6161602735519409, 0.5875178575515747], "cos_apical_negg": [0.5948014855384827, 0.6161602735519409, 0.5875178575515747], "cos_Ac_negg": [0.5948014855384827, 0.6161602735519409, 0.5875178575515747], "r_norm": [0.23586490750312805, 0.23370595276355743, 0.23451149463653564], "ldr": {"dL_sdil": -0.006271377205848694, "dL_bp": -0.007217004895210266, "ratio": 0.8689722810096526}}, {"step": 2300, "epoch": 5, "train_loss": 0.025918463245034218, "cos_r_negg": [0.5914753079414368, 0.618352472782135, 0.5954389572143555], "cos_innovation_negg": [0.5914753079414368, 0.618352472782135, 0.5954389572143555], "cos_apical_negg": [0.5914753079414368, 0.618352472782135, 0.5954389572143555], "cos_Ac_negg": [0.5914753079414368, 0.618352472782135, 0.5954389572143555], "r_norm": [0.23042558133602142, 0.22863107919692993, 0.2271466851234436]}, {"epoch_end": 5, "step": 2346, "test_acc": 0.9668, "test_loss": 0.1296664016723633}, {"step": 2350, "epoch": 6, "train_loss": 0.030179841443896294, "cos_r_negg": [0.5967842936515808, 0.6194745302200317, 0.5919294357299805], "cos_innovation_negg": [0.5967842936515808, 0.6194745302200317, 0.5919294357299805], "cos_apical_negg": [0.5967842936515808, 0.6194745302200317, 0.5919294357299805], "cos_Ac_negg": [0.5967842936515808, 0.6194745302200317, 0.5919294357299805], "r_norm": [0.23761086165905, 0.23442712426185608, 0.2342541366815567]}, {"step": 2400, "epoch": 6, "train_loss": 0.009492209181189537, "cos_r_negg": [0.6016510725021362, 0.6273598670959473, 0.5968002080917358], "cos_innovation_negg": [0.6016510725021362, 0.6273598670959473, 0.5968002080917358], "cos_apical_negg": [0.6016510725021362, 0.6273598670959473, 0.5968002080917358], "cos_Ac_negg": [0.6016510725021362, 0.6273598670959473, 0.5968002080917358], "r_norm": [0.23508545756340027, 0.23374991118907928, 0.2322363555431366]}, {"step": 2450, "epoch": 6, "train_loss": 0.012379295192658901, "cos_r_negg": [0.5889246463775635, 0.621046781539917, 0.6009576320648193], "cos_innovation_negg": [0.5889246463775635, 0.621046781539917, 0.6009576320648193], "cos_apical_negg": [0.5889246463775635, 0.621046781539917, 0.6009576320648193], "cos_Ac_negg": [0.5889246463775635, 0.621046781539917, 0.6009576320648193], "r_norm": [0.2146972417831421, 0.21452197432518005, 0.2138580083847046]}, {"step": 2500, "epoch": 6, "train_loss": 0.007920758798718452, "cos_r_negg": [0.5903428792953491, 0.6243600845336914, 0.5995810627937317], "cos_innovation_negg": [0.5903428792953491, 0.6243600845336914, 0.5995810627937317], "cos_apical_negg": [0.5903428792953491, 0.6243600845336914, 0.5995810627937317], "cos_Ac_negg": [0.5903428792953491, 0.6243600845336914, 0.5995810627937317], "r_norm": [0.2150765061378479, 0.21505454182624817, 0.2140389382839203], "ldr": {"dL_sdil": -0.0053649768233299255, "dL_bp": -0.007237352430820465, "ratio": 0.7412899778768578}}, {"step": 2550, "epoch": 6, "train_loss": 0.015546292066574097, "cos_r_negg": [0.5912970304489136, 0.6294369697570801, 0.6041755676269531], "cos_innovation_negg": [0.5912970304489136, 0.6294369697570801, 0.6041755676269531], "cos_apical_negg": [0.5912970304489136, 0.6294369697570801, 0.6041755676269531], "cos_Ac_negg": [0.5912970304489136, 0.6294369697570801, 0.6041755676269531], "r_norm": [0.1909637451171875, 0.18830755352973938, 0.1874294877052307]}, {"step": 2600, "epoch": 6, "train_loss": 0.012222237884998322, "cos_r_negg": [0.5850828289985657, 0.6211140155792236, 0.6042158603668213], "cos_innovation_negg": [0.5850828289985657, 0.6211140155792236, 0.6042158603668213], "cos_apical_negg": [0.5850828289985657, 0.6211140155792236, 0.6042158603668213], "cos_Ac_negg": [0.5850828289985657, 0.6211140155792236, 0.6042158603668213], "r_norm": [0.20255810022354126, 0.19920828938484192, 0.19814026355743408]}, {"step": 2650, "epoch": 6, "train_loss": 0.039999790489673615, "cos_r_negg": [0.5900393724441528, 0.6292356848716736, 0.6149412393569946], "cos_innovation_negg": [0.5900393724441528, 0.6292356848716736, 0.6149412393569946], "cos_apical_negg": [0.5900393724441528, 0.6292356848716736, 0.6149412393569946], "cos_Ac_negg": [0.5900393724441528, 0.6292356848716736, 0.6149412393569946], "r_norm": [0.2077878713607788, 0.20655818283557892, 0.2063518464565277]}, {"step": 2700, "epoch": 6, "train_loss": 0.0026352398563176394, "cos_r_negg": [0.5827772617340088, 0.6285635232925415, 0.6217774152755737], "cos_innovation_negg": [0.5827772617340088, 0.6285635232925415, 0.6217774152755737], "cos_apical_negg": [0.5827772617340088, 0.6285635232925415, 0.6217774152755737], "cos_Ac_negg": [0.5827772617340088, 0.6285635232925415, 0.6217774152755737], "r_norm": [0.2157292664051056, 0.21463735401630402, 0.21526111662387848]}, {"epoch_end": 6, "step": 2737, "test_acc": 0.9698, "test_loss": 0.12450692596435547}, {"step": 2750, "epoch": 7, "train_loss": 0.0010456736199557781, "cos_r_negg": [0.5876946449279785, 0.6276483535766602, 0.6213856339454651], "cos_innovation_negg": [0.5876946449279785, 0.6276483535766602, 0.6213856339454651], "cos_apical_negg": [0.5876946449279785, 0.6276483535766602, 0.6213856339454651], "cos_Ac_negg": [0.5876946449279785, 0.6276483535766602, 0.6213856339454651], "r_norm": [0.17490339279174805, 0.1732756495475769, 0.17379510402679443], "ldr": {"dL_sdil": -0.004224032163619995, "dL_bp": -0.005839265882968903, "ratio": 0.7233841116808913}}, {"step": 2800, "epoch": 7, "train_loss": 0.0045391833409667015, "cos_r_negg": [0.592957615852356, 0.6364576816558838, 0.6232614517211914], "cos_innovation_negg": [0.592957615852356, 0.6364576816558838, 0.6232614517211914], "cos_apical_negg": [0.592957615852356, 0.6364576816558838, 0.6232614517211914], "cos_Ac_negg": [0.592957615852356, 0.6364576816558838, 0.6232614517211914], "r_norm": [0.21607136726379395, 0.21277500689029694, 0.21436944603919983]}, {"step": 2850, "epoch": 7, "train_loss": 0.020631087943911552, "cos_r_negg": [0.595107913017273, 0.6380524635314941, 0.6282139420509338], "cos_innovation_negg": [0.595107913017273, 0.6380524635314941, 0.6282139420509338], "cos_apical_negg": [0.595107913017273, 0.6380524635314941, 0.6282139420509338], "cos_Ac_negg": [0.595107913017273, 0.6380524635314941, 0.6282139420509338], "r_norm": [0.18889783322811127, 0.18748202919960022, 0.18781843781471252]}, {"step": 2900, "epoch": 7, "train_loss": 0.008803481236100197, "cos_r_negg": [0.5978991985321045, 0.6429473161697388, 0.6330722570419312], "cos_innovation_negg": [0.5978991985321045, 0.6429473161697388, 0.6330722570419312], "cos_apical_negg": [0.5978991985321045, 0.6429473161697388, 0.6330722570419312], "cos_Ac_negg": [0.5978991985321045, 0.6429473161697388, 0.6330722570419312], "r_norm": [0.20737211406230927, 0.20499613881111145, 0.20534135401248932]}, {"step": 2950, "epoch": 7, "train_loss": 0.007925020530819893, "cos_r_negg": [0.5857475996017456, 0.6329107284545898, 0.6334774494171143], "cos_innovation_negg": [0.5857475996017456, 0.6329107284545898, 0.6334774494171143], "cos_apical_negg": [0.5857475996017456, 0.6329107284545898, 0.6334774494171143], "cos_Ac_negg": [0.5857475996017456, 0.6329107284545898, 0.6334774494171143], "r_norm": [0.19524097442626953, 0.19348791241645813, 0.192993625998497]}, {"step": 3000, "epoch": 7, "train_loss": 0.0038076424971222878, "cos_r_negg": [0.5903079509735107, 0.6430966258049011, 0.6327550411224365], "cos_innovation_negg": [0.5903079509735107, 0.6430966258049011, 0.6327550411224365], "cos_apical_negg": [0.5903079509735107, 0.6430966258049011, 0.6327550411224365], "cos_Ac_negg": [0.5903079509735107, 0.6430966258049011, 0.6327550411224365], "r_norm": [0.1909613311290741, 0.18970784544944763, 0.18949401378631592], "ldr": {"dL_sdil": -0.0051412805914878845, "dL_bp": -0.007529526948928833, "ratio": 0.6828158829047414}}, {"step": 3050, "epoch": 7, "train_loss": 0.0016820935998111963, "cos_r_negg": [0.5905494689941406, 0.6444506049156189, 0.6419535875320435], "cos_innovation_negg": [0.5905494689941406, 0.6444506049156189, 0.6419535875320435], "cos_apical_negg": [0.5905494689941406, 0.6444506049156189, 0.6419535875320435], "cos_Ac_negg": [0.5905494689941406, 0.6444506049156189, 0.6419535875320435], "r_norm": [0.187223881483078, 0.18650515377521515, 0.1877727210521698]}, {"step": 3100, "epoch": 7, "train_loss": 0.0017046871362254024, "cos_r_negg": [0.5912292003631592, 0.6383578777313232, 0.6389005780220032], "cos_innovation_negg": [0.5912292003631592, 0.6383578777313232, 0.6389005780220032], "cos_apical_negg": [0.5912292003631592, 0.6383578777313232, 0.6389005780220032], "cos_Ac_negg": [0.5912292003631592, 0.6383578777313232, 0.6389005780220032], "r_norm": [0.1633237600326538, 0.1618938148021698, 0.1630639135837555]}, {"epoch_end": 7, "step": 3128, "test_acc": 0.9693, "test_loss": 0.13391504650115968}, {"step": 3150, "epoch": 8, "train_loss": 0.01220756582915783, "cos_r_negg": [0.5866453647613525, 0.6397048234939575, 0.6408175826072693], "cos_innovation_negg": [0.5866453647613525, 0.6397048234939575, 0.6408175826072693], "cos_apical_negg": [0.5866453647613525, 0.6397048234939575, 0.6408175826072693], "cos_Ac_negg": [0.5866453647613525, 0.6397048234939575, 0.6408175826072693], "r_norm": [0.20577090978622437, 0.20323774218559265, 0.20256179571151733]}, {"step": 3200, "epoch": 8, "train_loss": 0.015854448080062866, "cos_r_negg": [0.5873883962631226, 0.6423099040985107, 0.6406865119934082], "cos_innovation_negg": [0.5873883962631226, 0.6423099040985107, 0.6406865119934082], "cos_apical_negg": [0.5873883962631226, 0.6423099040985107, 0.6406865119934082], "cos_Ac_negg": [0.5873883962631226, 0.6423099040985107, 0.6406865119934082], "r_norm": [0.19391030073165894, 0.19231054186820984, 0.192542165517807]}, {"step": 3250, "epoch": 8, "train_loss": 0.00592303741723299, "cos_r_negg": [0.5874128341674805, 0.6409730911254883, 0.6443665027618408], "cos_innovation_negg": [0.5874128341674805, 0.6409730911254883, 0.6443665027618408], "cos_apical_negg": [0.5874128341674805, 0.6409730911254883, 0.6443665027618408], "cos_Ac_negg": [0.5874128341674805, 0.6409730911254883, 0.6443665027618408], "r_norm": [0.20954379439353943, 0.20740677416324615, 0.20692391693592072], "ldr": {"dL_sdil": -0.007747858762741089, "dL_bp": -0.01186661422252655, "ratio": 0.6529123318118176}}, {"step": 3300, "epoch": 8, "train_loss": 0.006520576309412718, "cos_r_negg": [0.5842193961143494, 0.6396933794021606, 0.6441189050674438], "cos_innovation_negg": [0.5842193961143494, 0.6396933794021606, 0.6441189050674438], "cos_apical_negg": [0.5842193961143494, 0.6396933794021606, 0.6441189050674438], "cos_Ac_negg": [0.5842193961143494, 0.6396933794021606, 0.6441189050674438], "r_norm": [0.19906646013259888, 0.19688764214515686, 0.19866251945495605]}, {"step": 3350, "epoch": 8, "train_loss": 0.010459845885634422, "cos_r_negg": [0.5894404053688049, 0.6405558586120605, 0.6453732252120972], "cos_innovation_negg": [0.5894404053688049, 0.6405558586120605, 0.6453732252120972], "cos_apical_negg": [0.5894404053688049, 0.6405558586120605, 0.6453732252120972], "cos_Ac_negg": [0.5894404053688049, 0.6405558586120605, 0.6453732252120972], "r_norm": [0.21355968713760376, 0.21203957498073578, 0.2126498520374298]}, {"step": 3400, "epoch": 8, "train_loss": 0.009097838774323463, "cos_r_negg": [0.583891749382019, 0.6386954188346863, 0.6459414958953857], "cos_innovation_negg": [0.583891749382019, 0.6386954188346863, 0.6459414958953857], "cos_apical_negg": [0.583891749382019, 0.6386954188346863, 0.6459414958953857], "cos_Ac_negg": [0.583891749382019, 0.6386954188346863, 0.6459414958953857], "r_norm": [0.21383658051490784, 0.2115376740694046, 0.2117389440536499]}, {"step": 3450, "epoch": 8, "train_loss": 0.013778690248727798, "cos_r_negg": [0.5877927541732788, 0.6376104354858398, 0.6495387554168701], "cos_innovation_negg": [0.5877927541732788, 0.6376104354858398, 0.6495387554168701], "cos_apical_negg": [0.5877927541732788, 0.6376104354858398, 0.6495387554168701], "cos_Ac_negg": [0.5877927541732788, 0.6376104354858398, 0.6495387554168701], "r_norm": [0.2032446414232254, 0.2022097110748291, 0.2030501663684845]}, {"step": 3500, "epoch": 8, "train_loss": 0.012969301082193851, "cos_r_negg": [0.5818548202514648, 0.6368449926376343, 0.6515136361122131], "cos_innovation_negg": [0.5818548202514648, 0.6368449926376343, 0.6515136361122131], "cos_apical_negg": [0.5818548202514648, 0.6368449926376343, 0.6515136361122131], "cos_Ac_negg": [0.5818548202514648, 0.6368449926376343, 0.6515136361122131], "r_norm": [0.1928904801607132, 0.19001877307891846, 0.19078662991523743], "ldr": {"dL_sdil": -0.005356118083000183, "dL_bp": -0.008215487003326416, "ratio": 0.6519538136730681}}, {"epoch_end": 8, "step": 3519, "test_acc": 0.9707, "test_loss": 0.13156856966018676}, {"step": 3550, "epoch": 9, "train_loss": 0.008365225978195667, "cos_r_negg": [0.5858315229415894, 0.6421300768852234, 0.6559372544288635], "cos_innovation_negg": [0.5858315229415894, 0.6421300768852234, 0.6559372544288635], "cos_apical_negg": [0.5858315229415894, 0.6421300768852234, 0.6559372544288635], "cos_Ac_negg": [0.5858315229415894, 0.6421300768852234, 0.6559372544288635], "r_norm": [0.20125703513622284, 0.1980375498533249, 0.19839271903038025]}, {"step": 3600, "epoch": 9, "train_loss": 0.004163721576333046, "cos_r_negg": [0.5818840265274048, 0.6391579508781433, 0.6512700319290161], "cos_innovation_negg": [0.5818840265274048, 0.6391579508781433, 0.6512700319290161], "cos_apical_negg": [0.5818840265274048, 0.6391579508781433, 0.6512700319290161], "cos_Ac_negg": [0.5818840265274048, 0.6391579508781433, 0.6512700319290161], "r_norm": [0.20346075296401978, 0.2013006955385208, 0.2015729695558548]}, {"step": 3650, "epoch": 9, "train_loss": 0.0072011942975223064, "cos_r_negg": [0.58612060546875, 0.6452040076255798, 0.6542645692825317], "cos_innovation_negg": [0.58612060546875, 0.6452040076255798, 0.6542645692825317], "cos_apical_negg": [0.58612060546875, 0.6452040076255798, 0.6542645692825317], "cos_Ac_negg": [0.58612060546875, 0.6452040076255798, 0.6542645692825317], "r_norm": [0.20426571369171143, 0.2026892900466919, 0.20298320055007935]}, {"step": 3700, "epoch": 9, "train_loss": 0.009188925847411156, "cos_r_negg": [0.58309006690979, 0.6380341649055481, 0.6520888805389404], "cos_innovation_negg": [0.58309006690979, 0.6380341649055481, 0.6520888805389404], "cos_apical_negg": [0.58309006690979, 0.6380341649055481, 0.6520888805389404], "cos_Ac_negg": [0.58309006690979, 0.6380341649055481, 0.6520888805389404], "r_norm": [0.1986781805753708, 0.19639360904693604, 0.1967223435640335]}, {"step": 3750, "epoch": 9, "train_loss": 0.0010525946272537112, "cos_r_negg": [0.5796058177947998, 0.6385921239852905, 0.6552875638008118], "cos_innovation_negg": [0.5796058177947998, 0.6385921239852905, 0.6552875638008118], "cos_apical_negg": [0.5796058177947998, 0.6385921239852905, 0.6552875638008118], "cos_Ac_negg": [0.5796058177947998, 0.6385921239852905, 0.6552875638008118], "r_norm": [0.18747356534004211, 0.18508660793304443, 0.18633916974067688], "ldr": {"dL_sdil": -0.005741916596889496, "dL_bp": -0.009174227714538574, "ratio": 0.625874654036565}}, {"step": 3800, "epoch": 9, "train_loss": 0.002808848861604929, "cos_r_negg": [0.5797638893127441, 0.6380531787872314, 0.6550358533859253], "cos_innovation_negg": [0.5797638893127441, 0.6380531787872314, 0.6550358533859253], "cos_apical_negg": [0.5797638893127441, 0.6380531787872314, 0.6550358533859253], "cos_Ac_negg": [0.5797638893127441, 0.6380531787872314, 0.6550358533859253], "r_norm": [0.2012123465538025, 0.19828206300735474, 0.19915606081485748]}, {"step": 3850, "epoch": 9, "train_loss": 0.0016398265725001693, "cos_r_negg": [0.5864709615707397, 0.6429731249809265, 0.6599762439727783], "cos_innovation_negg": [0.5864709615707397, 0.6429731249809265, 0.6599762439727783], "cos_apical_negg": [0.5864709615707397, 0.6429731249809265, 0.6599762439727783], "cos_Ac_negg": [0.5864709615707397, 0.6429731249809265, 0.6599762439727783], "r_norm": [0.20660991966724396, 0.20368462800979614, 0.20481359958648682]}, {"step": 3900, "epoch": 9, "train_loss": 0.010912492871284485, "cos_r_negg": [0.5865564942359924, 0.6451301574707031, 0.6639076471328735], "cos_innovation_negg": [0.5865564942359924, 0.6451301574707031, 0.6639076471328735], "cos_apical_negg": [0.5865564942359924, 0.6451301574707031, 0.6639076471328735], "cos_Ac_negg": [0.5865564942359924, 0.6451301574707031, 0.6639076471328735], "r_norm": [0.20156991481781006, 0.19840535521507263, 0.19879092276096344]}, {"epoch_end": 9, "step": 3910, "test_acc": 0.9687, "test_loss": 0.1396755880355835}, {"step": 3950, "epoch": 10, "train_loss": 0.001801325473934412, "cos_r_negg": [0.5837472677230835, 0.6457388401031494, 0.6654669642448425], "cos_innovation_negg": [0.5837472677230835, 0.6457388401031494, 0.6654669642448425], "cos_apical_negg": [0.5837472677230835, 0.6457388401031494, 0.6654669642448425], "cos_Ac_negg": [0.5837472677230835, 0.6457388401031494, 0.6654669642448425], "r_norm": [0.20483875274658203, 0.20165368914604187, 0.20181261003017426]}, {"step": 4000, "epoch": 10, "train_loss": 0.0035876822657883167, "cos_r_negg": [0.582085132598877, 0.6433364152908325, 0.6627532243728638], "cos_innovation_negg": [0.582085132598877, 0.6433364152908325, 0.6627532243728638], "cos_apical_negg": [0.582085132598877, 0.6433364152908325, 0.6627532243728638], "cos_Ac_negg": [0.582085132598877, 0.6433364152908325, 0.6627532243728638], "r_norm": [0.21702535450458527, 0.21449218690395355, 0.21417111158370972], "ldr": {"dL_sdil": -0.007986009120941162, "dL_bp": -0.013906359672546387, "ratio": 0.5742702841712742}}, {"step": 4050, "epoch": 10, "train_loss": 0.0016625605057924986, "cos_r_negg": [0.5801330804824829, 0.6401721239089966, 0.6622391939163208], "cos_innovation_negg": [0.5801330804824829, 0.6401721239089966, 0.6622391939163208], "cos_apical_negg": [0.5801330804824829, 0.6401721239089966, 0.6622391939163208], "cos_Ac_negg": [0.5801330804824829, 0.6401721239089966, 0.6622391939163208], "r_norm": [0.21091443300247192, 0.2075100839138031, 0.2077319324016571]}, {"step": 4100, "epoch": 10, "train_loss": 0.001898560207337141, "cos_r_negg": [0.5830628871917725, 0.6439625024795532, 0.6650887727737427], "cos_innovation_negg": [0.5830628871917725, 0.6439625024795532, 0.6650887727737427], "cos_apical_negg": [0.5830628871917725, 0.6439625024795532, 0.6650887727737427], "cos_Ac_negg": [0.5830628871917725, 0.6439625024795532, 0.6650887727737427], "r_norm": [0.20665091276168823, 0.20300951600074768, 0.20310798287391663]}, {"step": 4150, "epoch": 10, "train_loss": 0.00033534382237121463, "cos_r_negg": [0.5834287405014038, 0.64290452003479, 0.6654666662216187], "cos_innovation_negg": [0.5834287405014038, 0.64290452003479, 0.6654666662216187], "cos_apical_negg": [0.5834287405014038, 0.64290452003479, 0.6654666662216187], "cos_Ac_negg": [0.5834287405014038, 0.64290452003479, 0.6654666662216187], "r_norm": [0.20601840317249298, 0.2031698226928711, 0.20371568202972412]}, {"step": 4200, "epoch": 10, "train_loss": 0.0011709406971931458, "cos_r_negg": [0.5832166075706482, 0.6425969004631042, 0.665043294429779], "cos_innovation_negg": [0.5832166075706482, 0.6425969004631042, 0.665043294429779], "cos_apical_negg": [0.5832166075706482, 0.6425969004631042, 0.665043294429779], "cos_Ac_negg": [0.5832166075706482, 0.6425969004631042, 0.665043294429779], "r_norm": [0.18991878628730774, 0.1871446967124939, 0.18802601099014282]}, {"step": 4250, "epoch": 10, "train_loss": 0.004689007997512817, "cos_r_negg": [0.5795853137969971, 0.6439259648323059, 0.664294421672821], "cos_innovation_negg": [0.5795853137969971, 0.6439259648323059, 0.664294421672821], "cos_apical_negg": [0.5795853137969971, 0.6439259648323059, 0.664294421672821], "cos_Ac_negg": [0.5795853137969971, 0.6439259648323059, 0.664294421672821], "r_norm": [0.2002757042646408, 0.19739732146263123, 0.19788289070129395], "ldr": {"dL_sdil": -0.007219210267066956, "dL_bp": -0.012639336287975311, "ratio": 0.5711700442637243}}, {"step": 4300, "epoch": 10, "train_loss": 0.001765637076459825, "cos_r_negg": [0.5781198740005493, 0.6453280448913574, 0.6673092246055603], "cos_innovation_negg": [0.5781198740005493, 0.6453280448913574, 0.6673092246055603], "cos_apical_negg": [0.5781198740005493, 0.6453280448913574, 0.6673092246055603], "cos_Ac_negg": [0.5781198740005493, 0.6453280448913574, 0.6673092246055603], "r_norm": [0.19356752932071686, 0.19138634204864502, 0.19159424304962158]}, {"epoch_end": 10, "step": 4301, "test_acc": 0.9699, "test_loss": 0.14492843971252442}, {"step": 4350, "epoch": 11, "train_loss": 0.0021812727209180593, "cos_r_negg": [0.579798698425293, 0.6459035277366638, 0.6690646409988403], "cos_innovation_negg": [0.579798698425293, 0.6459035277366638, 0.6690646409988403], "cos_apical_negg": [0.579798698425293, 0.6459035277366638, 0.6690646409988403], "cos_Ac_negg": [0.579798698425293, 0.6459035277366638, 0.6690646409988403], "r_norm": [0.19006434082984924, 0.18751077353954315, 0.1887262463569641]}, {"step": 4400, "epoch": 11, "train_loss": 0.0010056409519165754, "cos_r_negg": [0.5725233554840088, 0.6435121297836304, 0.6655387878417969], "cos_innovation_negg": [0.5725233554840088, 0.6435121297836304, 0.6655387878417969], "cos_apical_negg": [0.5725233554840088, 0.6435121297836304, 0.6655387878417969], "cos_Ac_negg": [0.5725233554840088, 0.6435121297836304, 0.6655387878417969], "r_norm": [0.1940477341413498, 0.19142918288707733, 0.19235356152057648]}, {"step": 4450, "epoch": 11, "train_loss": 0.0020795539021492004, "cos_r_negg": [0.5775647163391113, 0.6445392966270447, 0.66684889793396], "cos_innovation_negg": [0.5775647163391113, 0.6445392966270447, 0.66684889793396], "cos_apical_negg": [0.5775647163391113, 0.6445392966270447, 0.66684889793396], "cos_Ac_negg": [0.5775647163391113, 0.6445392966270447, 0.66684889793396], "r_norm": [0.18840859830379486, 0.1859579235315323, 0.18696486949920654]}, {"step": 4500, "epoch": 11, "train_loss": 0.0004823660710826516, "cos_r_negg": [0.5780872106552124, 0.6428213119506836, 0.6652305722236633], "cos_innovation_negg": [0.5780872106552124, 0.6428213119506836, 0.6652305722236633], "cos_apical_negg": [0.5780872106552124, 0.6428213119506836, 0.6652305722236633], "cos_Ac_negg": [0.5780872106552124, 0.6428213119506836, 0.6652305722236633], "r_norm": [0.19562821090221405, 0.1928991824388504, 0.19398891925811768], "ldr": {"dL_sdil": -0.006723120808601379, "dL_bp": -0.01107490062713623, "ratio": 0.6070592445884417}}, {"step": 4550, "epoch": 11, "train_loss": 0.0019139129435643554, "cos_r_negg": [0.5742108821868896, 0.641594409942627, 0.6678298711776733], "cos_innovation_negg": [0.5742108821868896, 0.641594409942627, 0.6678298711776733], "cos_apical_negg": [0.5742108821868896, 0.641594409942627, 0.6678298711776733], "cos_Ac_negg": [0.5742108821868896, 0.641594409942627, 0.6678298711776733], "r_norm": [0.18602152168750763, 0.18306244909763336, 0.18337664008140564]}, {"step": 4600, "epoch": 11, "train_loss": 0.00023855682229623199, "cos_r_negg": [0.5773031711578369, 0.6418589353561401, 0.6678142547607422], "cos_innovation_negg": [0.5773031711578369, 0.6418589353561401, 0.6678142547607422], "cos_apical_negg": [0.5773031711578369, 0.6418589353561401, 0.6678142547607422], "cos_Ac_negg": [0.5773031711578369, 0.6418589353561401, 0.6678142547607422], "r_norm": [0.1891641914844513, 0.18620739877223969, 0.18656717240810394]}, {"step": 4650, "epoch": 11, "train_loss": 0.0004666650202125311, "cos_r_negg": [0.5791573524475098, 0.6419100761413574, 0.667483389377594], "cos_innovation_negg": [0.5791573524475098, 0.6419100761413574, 0.667483389377594], "cos_apical_negg": [0.5791573524475098, 0.6419100761413574, 0.667483389377594], "cos_Ac_negg": [0.5791573524475098, 0.6419100761413574, 0.667483389377594], "r_norm": [0.19347336888313293, 0.19106516242027283, 0.19123101234436035]}, {"epoch_end": 11, "step": 4692, "test_acc": 0.9713, "test_loss": 0.14625187883377075}, {"step": 4700, "epoch": 12, "train_loss": 0.0009813150390982628, "cos_r_negg": [0.5778091549873352, 0.6436240673065186, 0.6693097352981567], "cos_innovation_negg": [0.5778091549873352, 0.6436240673065186, 0.6693097352981567], "cos_apical_negg": [0.5778091549873352, 0.6436240673065186, 0.6693097352981567], "cos_Ac_negg": [0.5778091549873352, 0.6436240673065186, 0.6693097352981567], "r_norm": [0.19017845392227173, 0.18806084990501404, 0.1885332465171814]}, {"step": 4750, "epoch": 12, "train_loss": 0.0010130723239853978, "cos_r_negg": [0.5783505439758301, 0.6435510516166687, 0.6665260791778564], "cos_innovation_negg": [0.5783505439758301, 0.6435510516166687, 0.6665260791778564], "cos_apical_negg": [0.5783505439758301, 0.6435510516166687, 0.6665260791778564], "cos_Ac_negg": [0.5783505439758301, 0.6435510516166687, 0.6665260791778564], "r_norm": [0.19416439533233643, 0.1919347643852234, 0.19243267178535461], "ldr": {"dL_sdil": -0.006809882819652557, "dL_bp": -0.011580392718315125, "ratio": 0.5880528394242016}}, {"step": 4800, "epoch": 12, "train_loss": 0.00040463305776938796, "cos_r_negg": [0.5777714252471924, 0.6423109769821167, 0.667091965675354], "cos_innovation_negg": [0.5777714252471924, 0.6423109769821167, 0.667091965675354], "cos_apical_negg": [0.5777714252471924, 0.6423109769821167, 0.667091965675354], "cos_Ac_negg": [0.5777714252471924, 0.6423109769821167, 0.667091965675354], "r_norm": [0.19179803133010864, 0.18920505046844482, 0.1899864673614502]}, {"step": 4850, "epoch": 12, "train_loss": 0.00046731403563171625, "cos_r_negg": [0.5773111581802368, 0.6416482329368591, 0.6670532822608948], "cos_innovation_negg": [0.5773111581802368, 0.6416482329368591, 0.6670532822608948], "cos_apical_negg": [0.5773111581802368, 0.6416482329368591, 0.6670532822608948], "cos_Ac_negg": [0.5773111581802368, 0.6416482329368591, 0.6670532822608948], "r_norm": [0.20203225314617157, 0.19956117868423462, 0.19995805621147156]}, {"step": 4900, "epoch": 12, "train_loss": 0.003285549581050873, "cos_r_negg": [0.5785969495773315, 0.6416221857070923, 0.6682823896408081], "cos_innovation_negg": [0.5785969495773315, 0.6416221857070923, 0.6682823896408081], "cos_apical_negg": [0.5785969495773315, 0.6416221857070923, 0.6682823896408081], "cos_Ac_negg": [0.5785969495773315, 0.6416221857070923, 0.6682823896408081], "r_norm": [0.20037418603897095, 0.19756051898002625, 0.19804002344608307]}, {"step": 4950, "epoch": 12, "train_loss": 0.0006839468842372298, "cos_r_negg": [0.575412392616272, 0.6401057243347168, 0.6687971353530884], "cos_innovation_negg": [0.575412392616272, 0.6401057243347168, 0.6687971353530884], "cos_apical_negg": [0.575412392616272, 0.6401057243347168, 0.6687971353530884], "cos_Ac_negg": [0.575412392616272, 0.6401057243347168, 0.6687971353530884], "r_norm": [0.20032697916030884, 0.19748103618621826, 0.19753497838974]}, {"step": 5000, "epoch": 12, "train_loss": 0.0006841165595687926, "cos_r_negg": [0.5796067118644714, 0.6423676013946533, 0.6711455583572388], "cos_innovation_negg": [0.5796067118644714, 0.6423676013946533, 0.6711455583572388], "cos_apical_negg": [0.5796067118644714, 0.6423676013946533, 0.6711455583572388], "cos_Ac_negg": [0.5796067118644714, 0.6423676013946533, 0.6711455583572388], "r_norm": [0.19935135543346405, 0.19666850566864014, 0.19714966416358948], "ldr": {"dL_sdil": -0.007333144545555115, "dL_bp": -0.01297459751367569, "ratio": 0.5651924491550291}}, {"step": 5050, "epoch": 12, "train_loss": 0.00043240224476903677, "cos_r_negg": [0.5794644355773926, 0.641514241695404, 0.6702879667282104], "cos_innovation_negg": [0.5794644355773926, 0.641514241695404, 0.6702879667282104], "cos_apical_negg": [0.5794644355773926, 0.641514241695404, 0.6702879667282104], "cos_Ac_negg": [0.5794644355773926, 0.641514241695404, 0.6702879667282104], "r_norm": [0.1879897117614746, 0.18518413603305817, 0.18605995178222656]}, {"epoch_end": 12, "step": 5083, "test_acc": 0.9717, "test_loss": 0.15002384853363038}, {"step": 5100, "epoch": 13, "train_loss": 0.0006865583127364516, "cos_r_negg": [0.5800986289978027, 0.642423689365387, 0.6703341007232666], "cos_innovation_negg": [0.5800986289978027, 0.642423689365387, 0.6703341007232666], "cos_apical_negg": [0.5800986289978027, 0.642423689365387, 0.6703341007232666], "cos_Ac_negg": [0.5800986289978027, 0.642423689365387, 0.6703341007232666], "r_norm": [0.19050440192222595, 0.18778377771377563, 0.18843460083007812]}, {"step": 5150, "epoch": 13, "train_loss": 0.0009239465580321848, "cos_r_negg": [0.579863429069519, 0.6424000859260559, 0.6710273027420044], "cos_innovation_negg": [0.579863429069519, 0.6424000859260559, 0.6710273027420044], "cos_apical_negg": [0.579863429069519, 0.6424000859260559, 0.6710273027420044], "cos_Ac_negg": [0.579863429069519, 0.6424000859260559, 0.6710273027420044], "r_norm": [0.19122929871082306, 0.18858855962753296, 0.18912187218666077]}, {"step": 5200, "epoch": 13, "train_loss": 0.0013198324013501406, "cos_r_negg": [0.5791923999786377, 0.6415884494781494, 0.6702650785446167], "cos_innovation_negg": [0.5791923999786377, 0.6415884494781494, 0.6702650785446167], "cos_apical_negg": [0.5791923999786377, 0.6415884494781494, 0.6702650785446167], "cos_Ac_negg": [0.5791923999786377, 0.6415884494781494, 0.6702650785446167], "r_norm": [0.19299572706222534, 0.19033202528953552, 0.1908753216266632]}, {"step": 5250, "epoch": 13, "train_loss": 0.00033962895395234227, "cos_r_negg": [0.5768936276435852, 0.6408150792121887, 0.670562207698822], "cos_innovation_negg": [0.5768936276435852, 0.6408150792121887, 0.670562207698822], "cos_apical_negg": [0.5768936276435852, 0.6408150792121887, 0.670562207698822], "cos_Ac_negg": [0.5768936276435852, 0.6408150792121887, 0.670562207698822], "r_norm": [0.19180870056152344, 0.18902021646499634, 0.18946868181228638], "ldr": {"dL_sdil": -0.006736628711223602, "dL_bp": -0.011798813939094543, "ratio": 0.5709581273167005}}, {"step": 5300, "epoch": 13, "train_loss": 0.0008334902231581509, "cos_r_negg": [0.5813297629356384, 0.6432279348373413, 0.6714141964912415], "cos_innovation_negg": [0.5813297629356384, 0.6432279348373413, 0.6714141964912415], "cos_apical_negg": [0.5813297629356384, 0.6432279348373413, 0.6714141964912415], "cos_Ac_negg": [0.5813297629356384, 0.6432279348373413, 0.6714141964912415], "r_norm": [0.19164404273033142, 0.18900665640830994, 0.18945330381393433]}, {"step": 5350, "epoch": 13, "train_loss": 0.0004194802022539079, "cos_r_negg": [0.5816265940666199, 0.643143892288208, 0.6724116802215576], "cos_innovation_negg": [0.5816265940666199, 0.643143892288208, 0.6724116802215576], "cos_apical_negg": [0.5816265940666199, 0.643143892288208, 0.6724116802215576], "cos_Ac_negg": [0.5816265940666199, 0.643143892288208, 0.6724116802215576], "r_norm": [0.20242296159267426, 0.20001596212387085, 0.20018336176872253]}, {"step": 5400, "epoch": 13, "train_loss": 0.001013430068269372, "cos_r_negg": [0.5793282389640808, 0.6411888003349304, 0.6730919480323792], "cos_innovation_negg": [0.5793282389640808, 0.6411888003349304, 0.6730919480323792], "cos_apical_negg": [0.5793282389640808, 0.6411888003349304, 0.6730919480323792], "cos_Ac_negg": [0.5793282389640808, 0.6411888003349304, 0.6730919480323792], "r_norm": [0.1946016252040863, 0.19186848402023315, 0.19223544001579285]}, {"step": 5450, "epoch": 13, "train_loss": 0.0005519220721907914, "cos_r_negg": [0.5785496234893799, 0.6423965096473694, 0.6732670068740845], "cos_innovation_negg": [0.5785496234893799, 0.6423965096473694, 0.6732670068740845], "cos_apical_negg": [0.5785496234893799, 0.6423965096473694, 0.6732670068740845], "cos_Ac_negg": [0.5785496234893799, 0.6423965096473694, 0.6732670068740845], "r_norm": [0.19343209266662598, 0.19101935625076294, 0.19153395295143127]}, {"epoch_end": 13, "step": 5474, "test_acc": 0.971, "test_loss": 0.1520557725906372}, {"step": 5500, "epoch": 14, "train_loss": 0.0006306613213382661, "cos_r_negg": [0.5773701667785645, 0.6420943737030029, 0.6731268167495728], "cos_innovation_negg": [0.5773701667785645, 0.6420943737030029, 0.6731268167495728], "cos_apical_negg": [0.5773701667785645, 0.6420943737030029, 0.6731268167495728], "cos_Ac_negg": [0.5773701667785645, 0.6420943737030029, 0.6731268167495728], "r_norm": [0.1897062063217163, 0.18694648146629333, 0.18750321865081787], "ldr": {"dL_sdil": -0.006870828568935394, "dL_bp": -0.012269079685211182, "ratio": 0.5600117323565276}}, {"step": 5550, "epoch": 14, "train_loss": 0.0004125842824578285, "cos_r_negg": [0.578131914138794, 0.64265376329422, 0.6734753847122192], "cos_innovation_negg": [0.578131914138794, 0.64265376329422, 0.6734753847122192], "cos_apical_negg": [0.578131914138794, 0.64265376329422, 0.6734753847122192], "cos_Ac_negg": [0.578131914138794, 0.64265376329422, 0.6734753847122192], "r_norm": [0.19237679243087769, 0.1895274519920349, 0.1901492178440094]}, {"step": 5600, "epoch": 14, "train_loss": 0.0014989023329690099, "cos_r_negg": [0.5790126323699951, 0.640927791595459, 0.672573447227478], "cos_innovation_negg": [0.5790126323699951, 0.640927791595459, 0.672573447227478], "cos_apical_negg": [0.5790126323699951, 0.640927791595459, 0.672573447227478], "cos_Ac_negg": [0.5790126323699951, 0.640927791595459, 0.672573447227478], "r_norm": [0.19329839944839478, 0.19058728218078613, 0.19092315435409546]}, {"step": 5650, "epoch": 14, "train_loss": 0.001080594607628882, "cos_r_negg": [0.5781686902046204, 0.6416630744934082, 0.6718908548355103], "cos_innovation_negg": [0.5781686902046204, 0.6416630744934082, 0.6718908548355103], "cos_apical_negg": [0.5781686902046204, 0.6416630744934082, 0.6718908548355103], "cos_Ac_negg": [0.5781686902046204, 0.6416630744934082, 0.6718908548355103], "r_norm": [0.1907031536102295, 0.18800823390483856, 0.1885772943496704]}, {"step": 5700, "epoch": 14, "train_loss": 0.00011184478353243321, "cos_r_negg": [0.5795497894287109, 0.641410768032074, 0.671786904335022], "cos_innovation_negg": [0.5795497894287109, 0.641410768032074, 0.671786904335022], "cos_apical_negg": [0.5795497894287109, 0.641410768032074, 0.671786904335022], "cos_Ac_negg": [0.5795497894287109, 0.641410768032074, 0.671786904335022], "r_norm": [0.18943551182746887, 0.18670538067817688, 0.18724828958511353]}, {"step": 5750, "epoch": 14, "train_loss": 0.00032396690221503377, "cos_r_negg": [0.5784250497817993, 0.6426033973693848, 0.6735066175460815], "cos_innovation_negg": [0.5784250497817993, 0.6426033973693848, 0.6735066175460815], "cos_apical_negg": [0.5784250497817993, 0.6426033973693848, 0.6735066175460815], "cos_Ac_negg": [0.5784250497817993, 0.6426033973693848, 0.6735066175460815], "r_norm": [0.1911245733499527, 0.18850651383399963, 0.18913917243480682], "ldr": {"dL_sdil": -0.007120877504348755, "dL_bp": -0.01304759830236435, "ratio": 0.5457615523815126}}, {"step": 5800, "epoch": 14, "train_loss": 0.0012019448913633823, "cos_r_negg": [0.5765959024429321, 0.6410543918609619, 0.6724869012832642], "cos_innovation_negg": [0.5765959024429321, 0.6410543918609619, 0.6724869012832642], "cos_apical_negg": [0.5765959024429321, 0.6410543918609619, 0.6724869012832642], "cos_Ac_negg": [0.5765959024429321, 0.6410543918609619, 0.6724869012832642], "r_norm": [0.18930310010910034, 0.1865515112876892, 0.1870606243610382]}, {"step": 5850, "epoch": 14, "train_loss": 0.0005424765404313803, "cos_r_negg": [0.5765285491943359, 0.6398369073867798, 0.6698485016822815], "cos_innovation_negg": [0.5765285491943359, 0.6398369073867798, 0.6698485016822815], "cos_apical_negg": [0.5765285491943359, 0.6398369073867798, 0.6698485016822815], "cos_Ac_negg": [0.5765285491943359, 0.6398369073867798, 0.6698485016822815], "r_norm": [0.19290603697299957, 0.1901761144399643, 0.19054442644119263]}, {"epoch_end": 14, "step": 5865, "test_acc": 0.9718, "test_loss": 0.15543756198883057}, {"step": 5900, "epoch": 15, "train_loss": 0.00024590262910351157, "cos_r_negg": [0.5785300731658936, 0.6405705809593201, 0.6705085039138794], "cos_innovation_negg": [0.5785300731658936, 0.6405705809593201, 0.6705085039138794], "cos_apical_negg": [0.5785300731658936, 0.6405705809593201, 0.6705085039138794], "cos_Ac_negg": [0.5785300731658936, 0.6405705809593201, 0.6705085039138794], "r_norm": [0.19304871559143066, 0.1903374195098877, 0.19072341918945312]}, {"step": 5950, "epoch": 15, "train_loss": 0.000682923651766032, "cos_r_negg": [0.5779988765716553, 0.6413237452507019, 0.6720834970474243], "cos_innovation_negg": [0.5779988765716553, 0.6413237452507019, 0.6720834970474243], "cos_apical_negg": [0.5779988765716553, 0.6413237452507019, 0.6720834970474243], "cos_Ac_negg": [0.5779988765716553, 0.6413237452507019, 0.6720834970474243], "r_norm": [0.18698656558990479, 0.18436086177825928, 0.1847958117723465]}, {"step": 6000, "epoch": 15, "train_loss": 0.00042269559344276786, "cos_r_negg": [0.578698992729187, 0.6402860879898071, 0.6706987619400024], "cos_innovation_negg": [0.578698992729187, 0.6402860879898071, 0.6706987619400024], "cos_apical_negg": [0.578698992729187, 0.6402860879898071, 0.6706987619400024], "cos_Ac_negg": [0.578698992729187, 0.6402860879898071, 0.6706987619400024], "r_norm": [0.18970437347888947, 0.18703682720661163, 0.18750843405723572], "ldr": {"dL_sdil": -0.0072825998067855835, "dL_bp": -0.013570696115493774, "ratio": 0.5366415801228487}}, {"step": 6050, "epoch": 15, "train_loss": 0.0009052111417986453, "cos_r_negg": [0.5771505832672119, 0.6387054920196533, 0.6694978475570679], "cos_innovation_negg": [0.5771505832672119, 0.6387054920196533, 0.6694978475570679], "cos_apical_negg": [0.5771505832672119, 0.6387054920196533, 0.6694978475570679], "cos_Ac_negg": [0.5771505832672119, 0.6387054920196533, 0.6694978475570679], "r_norm": [0.18927635252475739, 0.18660035729408264, 0.18714787065982819]}, {"step": 6100, "epoch": 15, "train_loss": 0.0014263016637414694, "cos_r_negg": [0.577475905418396, 0.6394616961479187, 0.669075071811676], "cos_innovation_negg": [0.577475905418396, 0.6394616961479187, 0.669075071811676], "cos_apical_negg": [0.577475905418396, 0.6394616961479187, 0.669075071811676], "cos_Ac_negg": [0.577475905418396, 0.6394616961479187, 0.669075071811676], "r_norm": [0.18955430388450623, 0.1867898404598236, 0.18738487362861633]}, {"step": 6150, "epoch": 15, "train_loss": 0.0008726827800273895, "cos_r_negg": [0.5789555311203003, 0.6404737830162048, 0.6704055070877075], "cos_innovation_negg": [0.5789555311203003, 0.6404737830162048, 0.6704055070877075], "cos_apical_negg": [0.5789555311203003, 0.6404737830162048, 0.6704055070877075], "cos_Ac_negg": [0.5789555311203003, 0.6404737830162048, 0.6704055070877075], "r_norm": [0.1911269873380661, 0.18853135406970978, 0.189109206199646]}, {"step": 6200, "epoch": 15, "train_loss": 0.0009533509728498757, "cos_r_negg": [0.5780545473098755, 0.6406347751617432, 0.6704813241958618], "cos_innovation_negg": [0.5780545473098755, 0.6406347751617432, 0.6704813241958618], "cos_apical_negg": [0.5780545473098755, 0.6406347751617432, 0.6704813241958618], "cos_Ac_negg": [0.5780545473098755, 0.6406347751617432, 0.6704813241958618], "r_norm": [0.19237200915813446, 0.18963725864887238, 0.1900343894958496]}, {"step": 6250, "epoch": 15, "train_loss": 0.00042144942563027143, "cos_r_negg": [0.5769616365432739, 0.6404335498809814, 0.6709768772125244], "cos_innovation_negg": [0.5769616365432739, 0.6404335498809814, 0.6709768772125244], "cos_apical_negg": [0.5769616365432739, 0.6404335498809814, 0.6709768772125244], "cos_Ac_negg": [0.5769616365432739, 0.6404335498809814, 0.6709768772125244], "r_norm": [0.1924067735671997, 0.18961450457572937, 0.18995633721351624], "ldr": {"dL_sdil": -0.00750916451215744, "dL_bp": -0.014312081038951874, "ratio": 0.5246731409443839}}, {"epoch_end": 15, "step": 6256, "test_acc": 0.9713, "test_loss": 0.15760864162445068}, {"step": 6300, "epoch": 16, "train_loss": 0.0006836053798906505, "cos_r_negg": [0.5765000581741333, 0.6415705680847168, 0.6718946695327759], "cos_innovation_negg": [0.5765000581741333, 0.6415705680847168, 0.6718946695327759], "cos_apical_negg": [0.5765000581741333, 0.6415705680847168, 0.6718946695327759], "cos_Ac_negg": [0.5765000581741333, 0.6415705680847168, 0.6718946695327759], "r_norm": [0.19371207058429718, 0.19089427590370178, 0.19120502471923828]}, {"step": 6350, "epoch": 16, "train_loss": 0.0002050266193691641, "cos_r_negg": [0.5776766538619995, 0.6408817768096924, 0.6717838048934937], "cos_innovation_negg": [0.5776766538619995, 0.6408817768096924, 0.6717838048934937], "cos_apical_negg": [0.5776766538619995, 0.6408817768096924, 0.6717838048934937], "cos_Ac_negg": [0.5776766538619995, 0.6408817768096924, 0.6717838048934937], "r_norm": [0.19619783759117126, 0.19354428350925446, 0.19396546483039856]}, {"step": 6400, "epoch": 16, "train_loss": 0.00032810436096042395, "cos_r_negg": [0.5788683891296387, 0.6404067277908325, 0.6714149713516235], "cos_innovation_negg": [0.5788683891296387, 0.6404067277908325, 0.6714149713516235], "cos_apical_negg": [0.5788683891296387, 0.6404067277908325, 0.6714149713516235], "cos_Ac_negg": [0.5788683891296387, 0.6404067277908325, 0.6714149713516235], "r_norm": [0.19279521703720093, 0.19026735424995422, 0.19085873663425446]}, {"step": 6450, "epoch": 16, "train_loss": 0.00010591735190246254, "cos_r_negg": [0.577681303024292, 0.6395716667175293, 0.6713933944702148], "cos_innovation_negg": [0.577681303024292, 0.6395716667175293, 0.6713933944702148], "cos_apical_negg": [0.577681303024292, 0.6395716667175293, 0.6713933944702148], "cos_Ac_negg": [0.577681303024292, 0.6395716667175293, 0.6713933944702148], "r_norm": [0.19000446796417236, 0.18744517862796783, 0.18788298964500427]}, {"step": 6500, "epoch": 16, "train_loss": 0.0014289491809904575, "cos_r_negg": [0.5778191089630127, 0.6392607688903809, 0.6708551645278931], "cos_innovation_negg": [0.5778191089630127, 0.6392607688903809, 0.6708551645278931], "cos_apical_negg": [0.5778191089630127, 0.6392607688903809, 0.6708551645278931], "cos_Ac_negg": [0.5778191089630127, 0.6392607688903809, 0.6708551645278931], "r_norm": [0.1894659548997879, 0.18684713542461395, 0.18731153011322021], "ldr": {"dL_sdil": -0.007261753082275391, "dL_bp": -0.013805337250232697, "ratio": 0.5260105530672922}}, {"step": 6550, "epoch": 16, "train_loss": 0.0005378465866670012, "cos_r_negg": [0.5787849426269531, 0.6400230526924133, 0.6729542016983032], "cos_innovation_negg": [0.5787849426269531, 0.6400230526924133, 0.6729542016983032], "cos_apical_negg": [0.5787849426269531, 0.6400230526924133, 0.6729542016983032], "cos_Ac_negg": [0.5787849426269531, 0.6400230526924133, 0.6729542016983032], "r_norm": [0.18946968019008636, 0.18663178384304047, 0.18712598085403442]}, {"step": 6600, "epoch": 16, "train_loss": 0.0003065415658056736, "cos_r_negg": [0.5779956579208374, 0.6396549940109253, 0.6720277667045593], "cos_innovation_negg": [0.5779956579208374, 0.6396549940109253, 0.6720277667045593], "cos_apical_negg": [0.5779956579208374, 0.6396549940109253, 0.6720277667045593], "cos_Ac_negg": [0.5779956579208374, 0.6396549940109253, 0.6720277667045593], "r_norm": [0.18738248944282532, 0.18463736772537231, 0.18517354130744934]}, {"epoch_end": 16, "step": 6647, "test_acc": 0.9716, "test_loss": 0.15967966499328612}, {"step": 6650, "epoch": 17, "train_loss": 0.00035405484959483147, "cos_r_negg": [0.5787295699119568, 0.6399439573287964, 0.6729344129562378], "cos_innovation_negg": [0.5787295699119568, 0.6399439573287964, 0.6729344129562378], "cos_apical_negg": [0.5787295699119568, 0.6399439573287964, 0.6729344129562378], "cos_Ac_negg": [0.5787295699119568, 0.6399439573287964, 0.6729344129562378], "r_norm": [0.19085925817489624, 0.1880280077457428, 0.18852412700653076]}, {"step": 6700, "epoch": 17, "train_loss": 0.0002382828388363123, "cos_r_negg": [0.579291582107544, 0.6404183506965637, 0.6735513210296631], "cos_innovation_negg": [0.579291582107544, 0.6404183506965637, 0.6735513210296631], "cos_apical_negg": [0.579291582107544, 0.6404183506965637, 0.6735513210296631], "cos_Ac_negg": [0.579291582107544, 0.6404183506965637, 0.6735513210296631], "r_norm": [0.19304843246936798, 0.1902361959218979, 0.19069457054138184]}, {"step": 6750, "epoch": 17, "train_loss": 0.0006765665020793676, "cos_r_negg": [0.578021228313446, 0.6402124166488647, 0.674172043800354], "cos_innovation_negg": [0.578021228313446, 0.6402124166488647, 0.674172043800354], "cos_apical_negg": [0.578021228313446, 0.6402124166488647, 0.674172043800354], "cos_Ac_negg": [0.578021228313446, 0.6402124166488647, 0.674172043800354], "r_norm": [0.19601547718048096, 0.193281888961792, 0.19356828927993774], "ldr": {"dL_sdil": -0.007917597889900208, "dL_bp": -0.015329346060752869, "ratio": 0.5164993900275581}}, {"step": 6800, "epoch": 17, "train_loss": 0.0007323332829400897, "cos_r_negg": [0.5776368975639343, 0.6409621238708496, 0.6735670566558838], "cos_innovation_negg": [0.5776368975639343, 0.6409621238708496, 0.6735670566558838], "cos_apical_negg": [0.5776368975639343, 0.6409621238708496, 0.6735670566558838], "cos_Ac_negg": [0.5776368975639343, 0.6409621238708496, 0.6735670566558838], "r_norm": [0.19396060705184937, 0.19128313660621643, 0.1917906105518341]}, {"step": 6850, "epoch": 17, "train_loss": 0.00038615401717834175, "cos_r_negg": [0.5768046379089355, 0.6398969888687134, 0.6726839542388916], "cos_innovation_negg": [0.5768046379089355, 0.6398969888687134, 0.6726839542388916], "cos_apical_negg": [0.5768046379089355, 0.6398969888687134, 0.6726839542388916], "cos_Ac_negg": [0.5768046379089355, 0.6398969888687134, 0.6726839542388916], "r_norm": [0.1923970878124237, 0.18962454795837402, 0.1901703178882599]}, {"step": 6900, "epoch": 17, "train_loss": 0.0006394105730578303, "cos_r_negg": [0.5784993767738342, 0.6388463973999023, 0.672380805015564], "cos_innovation_negg": [0.5784993767738342, 0.6388463973999023, 0.672380805015564], "cos_apical_negg": [0.5784993767738342, 0.6388463973999023, 0.672380805015564], "cos_Ac_negg": [0.5784993767738342, 0.6388463973999023, 0.672380805015564], "r_norm": [0.1898406744003296, 0.1869019716978073, 0.18752217292785645]}, {"step": 6950, "epoch": 17, "train_loss": 0.0008375168545171618, "cos_r_negg": [0.5794200897216797, 0.6396899819374084, 0.6732836961746216], "cos_innovation_negg": [0.5794200897216797, 0.6396899819374084, 0.6732836961746216], "cos_apical_negg": [0.5794200897216797, 0.6396899819374084, 0.6732836961746216], "cos_Ac_negg": [0.5794200897216797, 0.6396899819374084, 0.6732836961746216], "r_norm": [0.18868133425712585, 0.18582619726657867, 0.18646106123924255]}, {"step": 7000, "epoch": 17, "train_loss": 0.00011567586625460535, "cos_r_negg": [0.5788435935974121, 0.6395614147186279, 0.6744693517684937], "cos_innovation_negg": [0.5788435935974121, 0.6395614147186279, 0.6744693517684937], "cos_apical_negg": [0.5788435935974121, 0.6395614147186279, 0.6744693517684937], "cos_Ac_negg": [0.5788435935974121, 0.6395614147186279, 0.6744693517684937], "r_norm": [0.1855076104402542, 0.18267002701759338, 0.18327008187770844], "ldr": {"dL_sdil": -0.007222689688205719, "dL_bp": -0.013965465128421783, "ratio": 0.517182179167558}}, {"epoch_end": 17, "step": 7038, "test_acc": 0.9716, "test_loss": 0.1611394563674927}, {"step": 7050, "epoch": 18, "train_loss": 0.00027447042521089315, "cos_r_negg": [0.5783682465553284, 0.6392704248428345, 0.673374354839325], "cos_innovation_negg": [0.5783682465553284, 0.6392704248428345, 0.673374354839325], "cos_apical_negg": [0.5783682465553284, 0.6392704248428345, 0.673374354839325], "cos_Ac_negg": [0.5783682465553284, 0.6392704248428345, 0.673374354839325], "r_norm": [0.18603672087192535, 0.18332767486572266, 0.18383346498012543]}, {"step": 7100, "epoch": 18, "train_loss": 0.00028183619724586606, "cos_r_negg": [0.5781387090682983, 0.639603853225708, 0.6727410554885864], "cos_innovation_negg": [0.5781387090682983, 0.639603853225708, 0.6727410554885864], "cos_apical_negg": [0.5781387090682983, 0.639603853225708, 0.6727410554885864], "cos_Ac_negg": [0.5781387090682983, 0.639603853225708, 0.6727410554885864], "r_norm": [0.18623897433280945, 0.1834220588207245, 0.18398281931877136]}, {"step": 7150, "epoch": 18, "train_loss": 0.00033649831311777234, "cos_r_negg": [0.5778968334197998, 0.6396024227142334, 0.6737879514694214], "cos_innovation_negg": [0.5778968334197998, 0.6396024227142334, 0.6737879514694214], "cos_apical_negg": [0.5778968334197998, 0.6396024227142334, 0.6737879514694214], "cos_Ac_negg": [0.5778968334197998, 0.6396024227142334, 0.6737879514694214], "r_norm": [0.18947558104991913, 0.18657557666301727, 0.18713518977165222]}, {"step": 7200, "epoch": 18, "train_loss": 0.00046670809388160706, "cos_r_negg": [0.5784142017364502, 0.6392271518707275, 0.6731586456298828], "cos_innovation_negg": [0.5784142017364502, 0.6392271518707275, 0.6731586456298828], "cos_apical_negg": [0.5784142017364502, 0.6392271518707275, 0.6731586456298828], "cos_Ac_negg": [0.5784142017364502, 0.6392271518707275, 0.6731586456298828], "r_norm": [0.19152650237083435, 0.1887391209602356, 0.18932560086250305]}, {"step": 7250, "epoch": 18, "train_loss": 0.0004154551716055721, "cos_r_negg": [0.5781389474868774, 0.6388595700263977, 0.6738163232803345], "cos_innovation_negg": [0.5781389474868774, 0.6388595700263977, 0.6738163232803345], "cos_apical_negg": [0.5781389474868774, 0.6388595700263977, 0.6738163232803345], "cos_Ac_negg": [0.5781389474868774, 0.6388595700263977, 0.6738163232803345], "r_norm": [0.18920838832855225, 0.18633823096752167, 0.1869334876537323], "ldr": {"dL_sdil": -0.0076901912689208984, "dL_bp": -0.015084616839885712, "ratio": 0.5098035535504635}}, {"step": 7300, "epoch": 18, "train_loss": 0.00025237066438421607, "cos_r_negg": [0.5789194703102112, 0.6382718086242676, 0.6725468635559082], "cos_innovation_negg": [0.5789194703102112, 0.6382718086242676, 0.6725468635559082], "cos_apical_negg": [0.5789194703102112, 0.6382718086242676, 0.6725468635559082], "cos_Ac_negg": [0.5789194703102112, 0.6382718086242676, 0.6725468635559082], "r_norm": [0.18753387033939362, 0.1847638487815857, 0.18536600470542908]}, {"step": 7350, "epoch": 18, "train_loss": 0.0008477457449771464, "cos_r_negg": [0.5794123411178589, 0.6384925246238708, 0.6728180646896362], "cos_innovation_negg": [0.5794123411178589, 0.6384925246238708, 0.6728180646896362], "cos_apical_negg": [0.5794123411178589, 0.6384925246238708, 0.6728180646896362], "cos_Ac_negg": [0.5794123411178589, 0.6384925246238708, 0.6728180646896362], "r_norm": [0.18914490938186646, 0.18641002476215363, 0.18699605762958527]}, {"step": 7400, "epoch": 18, "train_loss": 0.0005081748240627348, "cos_r_negg": [0.5754134654998779, 0.6393170952796936, 0.6737634539604187], "cos_innovation_negg": [0.5754134654998779, 0.6393170952796936, 0.6737634539604187], "cos_apical_negg": [0.5754134654998779, 0.6393170952796936, 0.6737634539604187], "cos_Ac_negg": [0.5754134654998779, 0.6393170952796936, 0.6737634539604187], "r_norm": [0.1910279095172882, 0.18831104040145874, 0.1888488531112671]}, {"epoch_end": 18, "step": 7429, "test_acc": 0.9718, "test_loss": 0.16218714504241943}, {"step": 7450, "epoch": 19, "train_loss": 0.00014963520516175777, "cos_r_negg": [0.5762743949890137, 0.638745903968811, 0.6745541095733643], "cos_innovation_negg": [0.5762743949890137, 0.638745903968811, 0.6745541095733643], "cos_apical_negg": [0.5762743949890137, 0.638745903968811, 0.6745541095733643], "cos_Ac_negg": [0.5762743949890137, 0.638745903968811, 0.6745541095733643], "r_norm": [0.1896177977323532, 0.18685875833034515, 0.18737879395484924]}, {"step": 7500, "epoch": 19, "train_loss": 0.000283451663563028, "cos_r_negg": [0.5785921812057495, 0.6380590796470642, 0.6730641722679138], "cos_innovation_negg": [0.5785921812057495, 0.6380590796470642, 0.6730641722679138], "cos_apical_negg": [0.5785921812057495, 0.6380590796470642, 0.6730641722679138], "cos_Ac_negg": [0.5785921812057495, 0.6380590796470642, 0.6730641722679138], "r_norm": [0.18899798393249512, 0.18622273206710815, 0.18675431609153748], "ldr": {"dL_sdil": -0.00767911970615387, "dL_bp": -0.015145629644393921, "ratio": 0.5070188487671266}}, {"step": 7550, "epoch": 19, "train_loss": 0.0003964557545259595, "cos_r_negg": [0.574752926826477, 0.6383205652236938, 0.6746929883956909], "cos_innovation_negg": [0.574752926826477, 0.6383205652236938, 0.6746929883956909], "cos_apical_negg": [0.574752926826477, 0.6383205652236938, 0.6746929883956909], "cos_Ac_negg": [0.574752926826477, 0.6383205652236938, 0.6746929883956909], "r_norm": [0.18920782208442688, 0.18641342222690582, 0.18690736591815948]}, {"step": 7600, "epoch": 19, "train_loss": 0.0003488818765617907, "cos_r_negg": [0.573585033416748, 0.6381789445877075, 0.6740738749504089], "cos_innovation_negg": [0.573585033416748, 0.6381789445877075, 0.6740738749504089], "cos_apical_negg": [0.573585033416748, 0.6381789445877075, 0.6740738749504089], "cos_Ac_negg": [0.573585033416748, 0.6381789445877075, 0.6740738749504089], "r_norm": [0.18882064521312714, 0.18607982993125916, 0.18660160899162292]}, {"step": 7650, "epoch": 19, "train_loss": 0.0006052222452126443, "cos_r_negg": [0.5763521194458008, 0.638980507850647, 0.6739675998687744], "cos_innovation_negg": [0.5763521194458008, 0.638980507850647, 0.6739675998687744], "cos_apical_negg": [0.5763521194458008, 0.638980507850647, 0.6739675998687744], "cos_Ac_negg": [0.5763521194458008, 0.638980507850647, 0.6739675998687744], "r_norm": [0.18872112035751343, 0.18594515323638916, 0.1864684522151947]}, {"step": 7700, "epoch": 19, "train_loss": 0.0005076335510239005, "cos_r_negg": [0.5792561769485474, 0.6392600536346436, 0.67308509349823], "cos_innovation_negg": [0.5792561769485474, 0.6392600536346436, 0.67308509349823], "cos_apical_negg": [0.5792561769485474, 0.6392600536346436, 0.67308509349823], "cos_Ac_negg": [0.5792561769485474, 0.6392600536346436, 0.67308509349823], "r_norm": [0.1868249475955963, 0.18410003185272217, 0.1846197098493576]}, {"step": 7750, "epoch": 19, "train_loss": 0.00045577576383948326, "cos_r_negg": [0.5791441202163696, 0.6391366720199585, 0.6729779243469238], "cos_innovation_negg": [0.5791441202163696, 0.6391366720199585, 0.6729779243469238], "cos_apical_negg": [0.5791441202163696, 0.6391366720199585, 0.6729779243469238], "cos_Ac_negg": [0.5791441202163696, 0.6391366720199585, 0.6729779243469238], "r_norm": [0.18836525082588196, 0.18558944761753082, 0.18604837357997894], "ldr": {"dL_sdil": -0.007808975875377655, "dL_bp": -0.015600450336933136, "ratio": 0.5005609265580219}}, {"step": 7800, "epoch": 19, "train_loss": 0.0004446617094799876, "cos_r_negg": [0.5789731740951538, 0.6396101117134094, 0.6737980842590332], "cos_innovation_negg": [0.5789731740951538, 0.6396101117134094, 0.6737980842590332], "cos_apical_negg": [0.5789731740951538, 0.6396101117134094, 0.6737980842590332], "cos_Ac_negg": [0.5789731740951538, 0.6396101117134094, 0.6737980842590332], "r_norm": [0.187709242105484, 0.18496057391166687, 0.1854514479637146]}, {"epoch_end": 19, "step": 7820, "test_acc": 0.9715, "test_loss": 0.1645355073928833}, {"step": 7850, "epoch": 20, "train_loss": 0.00030412731575779617, "cos_r_negg": [0.5793260335922241, 0.6394457817077637, 0.6742372512817383], "cos_innovation_negg": [0.5793260335922241, 0.6394457817077637, 0.6742372512817383], "cos_apical_negg": [0.5793260335922241, 0.6394457817077637, 0.6742372512817383], "cos_Ac_negg": [0.5793260335922241, 0.6394457817077637, 0.6742372512817383], "r_norm": [0.18696963787078857, 0.18421778082847595, 0.18477855622768402]}, {"step": 7900, "epoch": 20, "train_loss": 0.0004169679305050522, "cos_r_negg": [0.5763202905654907, 0.6396397352218628, 0.6748013496398926], "cos_innovation_negg": [0.5763202905654907, 0.6396397352218628, 0.6748013496398926], "cos_apical_negg": [0.5763202905654907, 0.6396397352218628, 0.6748013496398926], "cos_Ac_negg": [0.5763202905654907, 0.6396397352218628, 0.6748013496398926], "r_norm": [0.1889667958021164, 0.18622884154319763, 0.1866796314716339]}, {"step": 7950, "epoch": 20, "train_loss": 0.00015641949721612036, "cos_r_negg": [0.5772417783737183, 0.6388957500457764, 0.6746581196784973], "cos_innovation_negg": [0.5772417783737183, 0.6388957500457764, 0.6746581196784973], "cos_apical_negg": [0.5772417783737183, 0.6388957500457764, 0.6746581196784973], "cos_Ac_negg": [0.5772417783737183, 0.6388957500457764, 0.6746581196784973], "r_norm": [0.19012832641601562, 0.18730930984020233, 0.18786199390888214]}, {"step": 8000, "epoch": 20, "train_loss": 0.00034836397389881313, "cos_r_negg": [0.576572060585022, 0.6393510103225708, 0.6752132177352905], "cos_innovation_negg": [0.576572060585022, 0.6393510103225708, 0.6752132177352905], "cos_apical_negg": [0.576572060585022, 0.6393510103225708, 0.6752132177352905], "cos_Ac_negg": [0.576572060585022, 0.6393510103225708, 0.6752132177352905], "r_norm": [0.1914433240890503, 0.18869906663894653, 0.18929049372673035], "ldr": {"dL_sdil": -0.008008532226085663, "dL_bp": -0.0160968080163002, "ratio": 0.4975230007077141}}, {"step": 8050, "epoch": 20, "train_loss": 0.0004983082180842757, "cos_r_negg": [0.5784183740615845, 0.63890540599823, 0.6738384962081909], "cos_innovation_negg": [0.5784183740615845, 0.63890540599823, 0.6738384962081909], "cos_apical_negg": [0.5784183740615845, 0.63890540599823, 0.6738384962081909], "cos_Ac_negg": [0.5784183740615845, 0.63890540599823, 0.6738384962081909], "r_norm": [0.19155672192573547, 0.18873000144958496, 0.18926188349723816]}, {"step": 8100, "epoch": 20, "train_loss": 0.0002325597743038088, "cos_r_negg": [0.5760648250579834, 0.6388427019119263, 0.6731445789337158], "cos_innovation_negg": [0.5760648250579834, 0.6388427019119263, 0.6731445789337158], "cos_apical_negg": [0.5760648250579834, 0.6388427019119263, 0.6731445789337158], "cos_Ac_negg": [0.5760648250579834, 0.6388427019119263, 0.6731445789337158], "r_norm": [0.18995754420757294, 0.18702512979507446, 0.18764187395572662]}, {"step": 8150, "epoch": 20, "train_loss": 0.0005227299989201128, "cos_r_negg": [0.5781093835830688, 0.6395500898361206, 0.6722992658615112], "cos_innovation_negg": [0.5781093835830688, 0.6395500898361206, 0.6722992658615112], "cos_apical_negg": [0.5781093835830688, 0.6395500898361206, 0.6722992658615112], "cos_Ac_negg": [0.5781093835830688, 0.6395500898361206, 0.6722992658615112], "r_norm": [0.18911099433898926, 0.18625833094120026, 0.18686163425445557]}, {"step": 8200, "epoch": 20, "train_loss": 0.00027412199415266514, "cos_r_negg": [0.5782171487808228, 0.6396183967590332, 0.6729393601417542], "cos_innovation_negg": [0.5782171487808228, 0.6396183967590332, 0.6729393601417542], "cos_apical_negg": [0.5782171487808228, 0.6396183967590332, 0.6729393601417542], "cos_Ac_negg": [0.5782171487808228, 0.6396183967590332, 0.6729393601417542], "r_norm": [0.18849530816078186, 0.18563777208328247, 0.18619082868099213]}, {"epoch_end": 20, "step": 8211, "test_acc": 0.9716, "test_loss": 0.16581540908813477}, {"step": 8250, "epoch": 21, "train_loss": 0.00043901349999941885, "cos_r_negg": [0.5774998664855957, 0.6394723057746887, 0.6733170747756958], "cos_innovation_negg": [0.5774998664855957, 0.6394723057746887, 0.6733170747756958], "cos_apical_negg": [0.5774998664855957, 0.6394723057746887, 0.6733170747756958], "cos_Ac_negg": [0.5774998664855957, 0.6394723057746887, 0.6733170747756958], "r_norm": [0.1890903115272522, 0.18620529770851135, 0.186717227101326], "ldr": {"dL_sdil": -0.00801774114370346, "dL_bp": -0.01622048020362854, "ratio": 0.4942973970591748}}, {"step": 8300, "epoch": 21, "train_loss": 0.000514088140334934, "cos_r_negg": [0.5782884359359741, 0.6392581462860107, 0.6735924482345581], "cos_innovation_negg": [0.5782884359359741, 0.6392581462860107, 0.6735924482345581], "cos_apical_negg": [0.5782884359359741, 0.6392581462860107, 0.6735924482345581], "cos_Ac_negg": [0.5782884359359741, 0.6392581462860107, 0.6735924482345581], "r_norm": [0.18732759356498718, 0.18458914756774902, 0.18518292903900146]}, {"step": 8350, "epoch": 21, "train_loss": 0.00045008573215454817, "cos_r_negg": [0.5793129205703735, 0.6398784518241882, 0.673973560333252], "cos_innovation_negg": [0.5793129205703735, 0.6398784518241882, 0.673973560333252], "cos_apical_negg": [0.5793129205703735, 0.6398784518241882, 0.673973560333252], "cos_Ac_negg": [0.5793129205703735, 0.6398784518241882, 0.673973560333252], "r_norm": [0.18680664896965027, 0.18413540720939636, 0.18468600511550903]}, {"step": 8400, "epoch": 21, "train_loss": 0.00018957926658913493, "cos_r_negg": [0.5786975026130676, 0.6392209529876709, 0.6742573976516724], "cos_innovation_negg": [0.5786975026130676, 0.6392209529876709, 0.6742573976516724], "cos_apical_negg": [0.5786975026130676, 0.6392209529876709, 0.6742573976516724], "cos_Ac_negg": [0.5786975026130676, 0.6392209529876709, 0.6742573976516724], "r_norm": [0.18537765741348267, 0.1825564205646515, 0.18319013714790344]}, {"step": 8450, "epoch": 21, "train_loss": 0.00015151046682149172, "cos_r_negg": [0.5775272846221924, 0.6387168169021606, 0.6741852760314941], "cos_innovation_negg": [0.5775272846221924, 0.6387168169021606, 0.6741852760314941], "cos_apical_negg": [0.5775272846221924, 0.6387168169021606, 0.6741852760314941], "cos_Ac_negg": [0.5775272846221924, 0.6387168169021606, 0.6741852760314941], "r_norm": [0.1871771663427353, 0.18429142236709595, 0.1849348545074463]}, {"step": 8500, "epoch": 21, "train_loss": 0.0004651687922887504, "cos_r_negg": [0.5758571028709412, 0.6389057040214539, 0.6748074889183044], "cos_innovation_negg": [0.5758571028709412, 0.6389057040214539, 0.6748074889183044], "cos_apical_negg": [0.5758571028709412, 0.6389057040214539, 0.6748074889183044], "cos_Ac_negg": [0.5758571028709412, 0.6389057040214539, 0.6748074889183044], "r_norm": [0.18963709473609924, 0.1867041140794754, 0.18722832202911377], "ldr": {"dL_sdil": -0.008045002818107605, "dL_bp": -0.01630319654941559, "ratio": 0.4934616836473084}}, {"step": 8550, "epoch": 21, "train_loss": 0.0002482909767422825, "cos_r_negg": [0.5769458413124084, 0.6391909122467041, 0.6736286282539368], "cos_innovation_negg": [0.5769458413124084, 0.6391909122467041, 0.6736286282539368], "cos_apical_negg": [0.5769458413124084, 0.6391909122467041, 0.6736286282539368], "cos_Ac_negg": [0.5769458413124084, 0.6391909122467041, 0.6736286282539368], "r_norm": [0.19062379002571106, 0.18769484758377075, 0.18823812901973724]}, {"step": 8600, "epoch": 21, "train_loss": 0.0005517169483937323, "cos_r_negg": [0.5768244862556458, 0.6395310163497925, 0.6735292673110962], "cos_innovation_negg": [0.5768244862556458, 0.6395310163497925, 0.6735292673110962], "cos_apical_negg": [0.5768244862556458, 0.6395310163497925, 0.6735292673110962], "cos_Ac_negg": [0.5768244862556458, 0.6395310163497925, 0.6735292673110962], "r_norm": [0.19067463278770447, 0.18793529272079468, 0.1884758174419403]}, {"epoch_end": 21, "step": 8602, "test_acc": 0.9717, "test_loss": 0.16728587074279785}, {"step": 8650, "epoch": 22, "train_loss": 0.00010312828817404807, "cos_r_negg": [0.5765217542648315, 0.6393990516662598, 0.6733987331390381], "cos_innovation_negg": [0.5765217542648315, 0.6393990516662598, 0.6733987331390381], "cos_apical_negg": [0.5765217542648315, 0.6393990516662598, 0.6733987331390381], "cos_Ac_negg": [0.5765217542648315, 0.6393990516662598, 0.6733987331390381], "r_norm": [0.19026055932044983, 0.18745511770248413, 0.1880102902650833]}, {"step": 8700, "epoch": 22, "train_loss": 0.00012435158714652061, "cos_r_negg": [0.5765522718429565, 0.6392253041267395, 0.6738529205322266], "cos_innovation_negg": [0.5765522718429565, 0.6392253041267395, 0.6738529205322266], "cos_apical_negg": [0.5765522718429565, 0.6392253041267395, 0.6738529205322266], "cos_Ac_negg": [0.5765522718429565, 0.6392253041267395, 0.6738529205322266], "r_norm": [0.19082759320735931, 0.18804407119750977, 0.1886894702911377]}, {"step": 8750, "epoch": 22, "train_loss": 0.0003034854307770729, "cos_r_negg": [0.5771274566650391, 0.6399524807929993, 0.6744510531425476], "cos_innovation_negg": [0.5771274566650391, 0.6399524807929993, 0.6744510531425476], "cos_apical_negg": [0.5771274566650391, 0.6399524807929993, 0.6744510531425476], "cos_Ac_negg": [0.5771274566650391, 0.6399524807929993, 0.6744510531425476], "r_norm": [0.18970917165279388, 0.1869012713432312, 0.18751563131809235], "ldr": {"dL_sdil": -0.008147850632667542, "dL_bp": -0.016648992896080017, "ratio": 0.48938999995524907}}, {"step": 8800, "epoch": 22, "train_loss": 0.00048770150169730186, "cos_r_negg": [0.5773370265960693, 0.6395368576049805, 0.6745785474777222], "cos_innovation_negg": [0.5773370265960693, 0.6395368576049805, 0.6745785474777222], "cos_apical_negg": [0.5773370265960693, 0.6395368576049805, 0.6745785474777222], "cos_Ac_negg": [0.5773370265960693, 0.6395368576049805, 0.6745785474777222], "r_norm": [0.187653049826622, 0.18477334082126617, 0.18540942668914795]}, {"step": 8850, "epoch": 22, "train_loss": 0.0002193791588069871, "cos_r_negg": [0.5775732398033142, 0.6395695209503174, 0.6749089956283569], "cos_innovation_negg": [0.5775732398033142, 0.6395695209503174, 0.6749089956283569], "cos_apical_negg": [0.5775732398033142, 0.6395695209503174, 0.6749089956283569], "cos_Ac_negg": [0.5775732398033142, 0.6395695209503174, 0.6749089956283569], "r_norm": [0.1850675493478775, 0.1822938621044159, 0.18291619420051575]}, {"step": 8900, "epoch": 22, "train_loss": 0.0003655984764918685, "cos_r_negg": [0.5774120092391968, 0.6393335461616516, 0.674842119216919], "cos_innovation_negg": [0.5774120092391968, 0.6393335461616516, 0.674842119216919], "cos_apical_negg": [0.5774120092391968, 0.6393335461616516, 0.674842119216919], "cos_Ac_negg": [0.5774120092391968, 0.6393335461616516, 0.674842119216919], "r_norm": [0.18698513507843018, 0.18415787816047668, 0.1847560852766037]}, {"step": 8950, "epoch": 22, "train_loss": 0.0003299964009784162, "cos_r_negg": [0.5773218870162964, 0.6394993662834167, 0.6752936840057373], "cos_innovation_negg": [0.5773218870162964, 0.6394993662834167, 0.6752936840057373], "cos_apical_negg": [0.5773218870162964, 0.6394993662834167, 0.6752936840057373], "cos_Ac_negg": [0.5773218870162964, 0.6394993662834167, 0.6752936840057373], "r_norm": [0.18754833936691284, 0.1847684532403946, 0.18540000915527344]}, {"epoch_end": 22, "step": 8993, "test_acc": 0.9719, "test_loss": 0.16839540176391601}, {"step": 9000, "epoch": 23, "train_loss": 0.00022772296506445855, "cos_r_negg": [0.5768519639968872, 0.6394088268280029, 0.6747658252716064], "cos_innovation_negg": [0.5768519639968872, 0.6394088268280029, 0.6747658252716064], "cos_apical_negg": [0.5768519639968872, 0.6394088268280029, 0.6747658252716064], "cos_Ac_negg": [0.5768519639968872, 0.6394088268280029, 0.6747658252716064], "r_norm": [0.18852803111076355, 0.18571104109287262, 0.18633770942687988], "ldr": {"dL_sdil": -0.008137457072734833, "dL_bp": -0.01674012839794159, "ratio": 0.48610481827220847}}, {"step": 9050, "epoch": 23, "train_loss": 0.00039757537888363004, "cos_r_negg": [0.5772574543952942, 0.6400966644287109, 0.6748083829879761], "cos_innovation_negg": [0.5772574543952942, 0.6400966644287109, 0.6748083829879761], "cos_apical_negg": [0.5772574543952942, 0.6400966644287109, 0.6748083829879761], "cos_Ac_negg": [0.5772574543952942, 0.6400966644287109, 0.6748083829879761], "r_norm": [0.18739785254001617, 0.1846676766872406, 0.18536067008972168]}, {"step": 9100, "epoch": 23, "train_loss": 0.00014316420129034668, "cos_r_negg": [0.5776441693305969, 0.6399005651473999, 0.6747811436653137], "cos_innovation_negg": [0.5776441693305969, 0.6399005651473999, 0.6747811436653137], "cos_apical_negg": [0.5776441693305969, 0.6399005651473999, 0.6747811436653137], "cos_Ac_negg": [0.5776441693305969, 0.6399005651473999, 0.6747811436653137], "r_norm": [0.18788674473762512, 0.18517589569091797, 0.18583062291145325]}, {"step": 9150, "epoch": 23, "train_loss": 0.00038942001992836595, "cos_r_negg": [0.5775863528251648, 0.6396216750144958, 0.6750222444534302], "cos_innovation_negg": [0.5775863528251648, 0.6396216750144958, 0.6750222444534302], "cos_apical_negg": [0.5775863528251648, 0.6396216750144958, 0.6750222444534302], "cos_Ac_negg": [0.5775863528251648, 0.6396216750144958, 0.6750222444534302], "r_norm": [0.189317524433136, 0.18657828867435455, 0.18715965747833252]}, {"step": 9200, "epoch": 23, "train_loss": 0.0006780464900657535, "cos_r_negg": [0.5783043503761292, 0.6398566365242004, 0.6751080751419067], "cos_innovation_negg": [0.5783043503761292, 0.6398566365242004, 0.6751080751419067], "cos_apical_negg": [0.5783043503761292, 0.6398566365242004, 0.6751080751419067], "cos_Ac_negg": [0.5783043503761292, 0.6398566365242004, 0.6751080751419067], "r_norm": [0.18851754069328308, 0.18579213321208954, 0.18636637926101685]}, {"step": 9250, "epoch": 23, "train_loss": 0.00023991052876226604, "cos_r_negg": [0.5775536894798279, 0.6392917037010193, 0.6757921576499939], "cos_innovation_negg": [0.5775536894798279, 0.6392917037010193, 0.6757921576499939], "cos_apical_negg": [0.5775536894798279, 0.6392917037010193, 0.6757921576499939], "cos_Ac_negg": [0.5775536894798279, 0.6392917037010193, 0.6757921576499939], "r_norm": [0.18768183887004852, 0.18488198518753052, 0.18543002009391785], "ldr": {"dL_sdil": -0.008142448961734772, "dL_bp": -0.01683904230594635, "ratio": 0.4835458462420656}}, {"step": 9300, "epoch": 23, "train_loss": 0.0001646615273784846, "cos_r_negg": [0.5778659582138062, 0.6396468877792358, 0.6760585308074951], "cos_innovation_negg": [0.5778659582138062, 0.6396468877792358, 0.6760585308074951], "cos_apical_negg": [0.5778659582138062, 0.6396468877792358, 0.6760585308074951], "cos_Ac_negg": [0.5778659582138062, 0.6396468877792358, 0.6760585308074951], "r_norm": [0.1876998245716095, 0.18492650985717773, 0.18542009592056274]}, {"step": 9350, "epoch": 23, "train_loss": 9.24638588912785e-05, "cos_r_negg": [0.5777623653411865, 0.6394279599189758, 0.6766988039016724], "cos_innovation_negg": [0.5777623653411865, 0.6394279599189758, 0.6766988039016724], "cos_apical_negg": [0.5777623653411865, 0.6394279599189758, 0.6766988039016724], "cos_Ac_negg": [0.5777623653411865, 0.6394279599189758, 0.6766988039016724], "r_norm": [0.18724724650382996, 0.18435470759868622, 0.18485525250434875]}, {"epoch_end": 23, "step": 9384, "test_acc": 0.9715, "test_loss": 0.169584246635437}, {"step": 9400, "epoch": 24, "train_loss": 0.00014438485959544778, "cos_r_negg": [0.5774277448654175, 0.6391093730926514, 0.6759103536605835], "cos_innovation_negg": [0.5774277448654175, 0.6391093730926514, 0.6759103536605835], "cos_apical_negg": [0.5774277448654175, 0.6391093730926514, 0.6759103536605835], "cos_Ac_negg": [0.5774277448654175, 0.6391093730926514, 0.6759103536605835], "r_norm": [0.1869356334209442, 0.18408414721488953, 0.18471907079219818]}, {"step": 9450, "epoch": 24, "train_loss": 0.00010408880916656926, "cos_r_negg": [0.5768346786499023, 0.6394554376602173, 0.6756951808929443], "cos_innovation_negg": [0.5768346786499023, 0.6394554376602173, 0.6756951808929443], "cos_apical_negg": [0.5768346786499023, 0.6394554376602173, 0.6756951808929443], "cos_Ac_negg": [0.5768346786499023, 0.6394554376602173, 0.6756951808929443], "r_norm": [0.18724502623081207, 0.1843583583831787, 0.18501389026641846]}, {"step": 9500, "epoch": 24, "train_loss": 0.00022216116485651582, "cos_r_negg": [0.5762655138969421, 0.639163076877594, 0.6758091449737549], "cos_innovation_negg": [0.5762655138969421, 0.639163076877594, 0.6758091449737549], "cos_apical_negg": [0.5762655138969421, 0.639163076877594, 0.6758091449737549], "cos_Ac_negg": [0.5762655138969421, 0.639163076877594, 0.6758091449737549], "r_norm": [0.1866132616996765, 0.18374481797218323, 0.1844094842672348], "ldr": {"dL_sdil": -0.00811118632555008, "dL_bp": -0.01682981848716736, "ratio": 0.4819532861709004}}, {"step": 9550, "epoch": 24, "train_loss": 0.00013690139167010784, "cos_r_negg": [0.5773353576660156, 0.6402164697647095, 0.6759779453277588], "cos_innovation_negg": [0.5773353576660156, 0.6402164697647095, 0.6759779453277588], "cos_apical_negg": [0.5773353576660156, 0.6402164697647095, 0.6759779453277588], "cos_Ac_negg": [0.5773353576660156, 0.6402164697647095, 0.6759779453277588], "r_norm": [0.1852986216545105, 0.18256866931915283, 0.18319405615329742]}, {"step": 9600, "epoch": 24, "train_loss": 0.00018698934582062066, "cos_r_negg": [0.5763930082321167, 0.6389181613922119, 0.6758542060852051], "cos_innovation_negg": [0.5763930082321167, 0.6389181613922119, 0.6758542060852051], "cos_apical_negg": [0.5763930082321167, 0.6389181613922119, 0.6758542060852051], "cos_Ac_negg": [0.5763930082321167, 0.6389181613922119, 0.6758542060852051], "r_norm": [0.1864422857761383, 0.1836596429347992, 0.1842782348394394]}, {"step": 9650, "epoch": 24, "train_loss": 0.0004146886058151722, "cos_r_negg": [0.5755079984664917, 0.6379781365394592, 0.6754927635192871], "cos_innovation_negg": [0.5755079984664917, 0.6379781365394592, 0.6754927635192871], "cos_apical_negg": [0.5755079984664917, 0.6379781365394592, 0.6754927635192871], "cos_Ac_negg": [0.5755079984664917, 0.6379781365394592, 0.6754927635192871], "r_norm": [0.18529295921325684, 0.18258237838745117, 0.18328863382339478]}, {"step": 9700, "epoch": 24, "train_loss": 0.0003425304894335568, "cos_r_negg": [0.5766454935073853, 0.6383209228515625, 0.6761559844017029], "cos_innovation_negg": [0.5766454935073853, 0.6383209228515625, 0.6761559844017029], "cos_apical_negg": [0.5766454935073853, 0.6383209228515625, 0.6761559844017029], "cos_Ac_negg": [0.5766454935073853, 0.6383209228515625, 0.6761559844017029], "r_norm": [0.18789638578891754, 0.18516871333122253, 0.18579384684562683]}, {"step": 9750, "epoch": 24, "train_loss": 0.0005492063937708735, "cos_r_negg": [0.576439619064331, 0.6386055946350098, 0.6767020225524902], "cos_innovation_negg": [0.576439619064331, 0.6386055946350098, 0.6767020225524902], "cos_apical_negg": [0.576439619064331, 0.6386055946350098, 0.6767020225524902], "cos_Ac_negg": [0.576439619064331, 0.6386055946350098, 0.6767020225524902], "r_norm": [0.18605361878871918, 0.18326155841350555, 0.18389558792114258], "ldr": {"dL_sdil": -0.008051231503486633, "dL_bp": -0.016686365008354187, "ratio": 0.4825036189401171}}, {"epoch_end": 24, "step": 9775, "test_acc": 0.9716, "test_loss": 0.1701682970046997}], "final": {"test_acc": 0.9716, "test_loss": 0.1701682970046997, "wall_s": 36.26105189323425, "cos_r_negg": [0.5761867761611938, 0.638395369052887, 0.6764992475509644], "cos_innovation_negg": [0.5761867761611938, 0.638395369052887, 0.6764992475509644], "cos_apical_negg": [0.5761867761611938, 0.638395369052887, 0.6764992475509644], "cos_Ac_negg": [0.5761867761611938, 0.638395369052887, 0.6764992475509644]}} \ No newline at end of file diff --git a/results/ep_original_v2_mnist_ep_w500_d1_s0.json b/results/ep_original_v2_mnist_ep_w500_d1_s0.json new file mode 100644 index 0000000..a08555b --- /dev/null +++ b/results/ep_original_v2_mnist_ep_w500_d1_s0.json @@ -0,0 +1 @@ +{"args": {"method": "ep", "dataset": "mnist", "depth": 1, "width": 500, "epochs": 25, "batch_size": 20, "train_examples": 50000, "max_batches": 0, "eta": null, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 0, "device": "cuda", "outdir": "results", "tag": "ep_original_v2_mnist_ep_w500_d1_s0"}, "resolved_protocol": {"beta": 0.5, "free_steps": 20, "nudge_steps": 4, "canonical_depth": true, "learning_rates": [0.1, 0.05], "persistent_particles": true, "train_examples": 50000}, "method_source": {"paper": "https://www.frontiersin.org/journals/computational-neuroscience/articles/10.3389/fncom.2017.00024/full", "code": "https://github.com/bscellier/Towards-a-Biologically-Plausible-Backprop", "protocol": "two-phase energy-gradient dynamics; persistent free particles; random beta sign"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.955, "test_loss": 0.07738499794006348, "train_loss": 0.01003960333764553}, {"epoch_end": 1, "test_acc": 0.9648, "test_loss": 0.06048058605194092, "train_loss": 0.0037819012068212032}, {"epoch_end": 2, "test_acc": 0.9658, "test_loss": 0.056952181243896484, "train_loss": 0.0026113467756658792}, {"epoch_end": 3, "test_acc": 0.9668, "test_loss": 0.057208501434326174, "train_loss": 0.0011136147659271955}, {"epoch_end": 4, "test_acc": 0.971, "test_loss": 0.050622566032409666, "train_loss": 0.00023063279513735324}, {"epoch_end": 5, "test_acc": 0.9687, "test_loss": 0.052987321662902834, "train_loss": 0.00016184779815375805}, {"epoch_end": 6, "test_acc": 0.9693, "test_loss": 0.05012830390930176, "train_loss": 0.000521468638908118}, {"epoch_end": 7, "test_acc": 0.9723, "test_loss": 0.05139166889190674, "train_loss": 6.770728759875055e-06}, {"epoch_end": 8, "test_acc": 0.9657, "test_loss": 0.06045001754760742, "train_loss": 0.00023175214300863445}, {"epoch_end": 9, "test_acc": 0.9677, "test_loss": 0.0564575044631958, "train_loss": 5.406913624028675e-05}, {"epoch_end": 10, "test_acc": 0.9681, "test_loss": 0.056905843353271486, "train_loss": 1.3698017653496208e-07}, {"epoch_end": 11, "test_acc": 0.9686, "test_loss": 0.05428710899353027, "train_loss": 0.0}, {"epoch_end": 12, "test_acc": 0.971, "test_loss": 0.04870379829406738, "train_loss": 0.0}, {"epoch_end": 13, "test_acc": 0.9742, "test_loss": 0.04683217372894287, "train_loss": 0.0}, {"epoch_end": 14, "test_acc": 0.97, "test_loss": 0.04981306781768799, "train_loss": 0.0}, {"epoch_end": 15, "test_acc": 0.973, "test_loss": 0.049349253273010256, "train_loss": 1.141531811299501e-05}, {"epoch_end": 16, "test_acc": 0.9707, "test_loss": 0.04930688552856445, "train_loss": 0.0}, {"epoch_end": 17, "test_acc": 0.9687, "test_loss": 0.05342885799407959, "train_loss": 0.0}, {"epoch_end": 18, "test_acc": 0.9725, "test_loss": 0.049415488052368166, "train_loss": 0.0}, {"epoch_end": 19, "test_acc": 0.9706, "test_loss": 0.050550705146789554, "train_loss": 0.0}, {"epoch_end": 20, "test_acc": 0.9713, "test_loss": 0.0508961033821106, "train_loss": 0.0}, {"epoch_end": 21, "test_acc": 0.9715, "test_loss": 0.04933317308425903, "train_loss": 0.0}, {"epoch_end": 22, "test_acc": 0.9706, "test_loss": 0.051107603645324705, "train_loss": 0.0}, {"epoch_end": 23, "test_acc": 0.9719, "test_loss": 0.050396513938903806, "train_loss": 0.0}, {"epoch_end": 24, "test_acc": 0.9723, "test_loss": 0.05007862377166748, "train_loss": 0.0}], "final": {"test_acc": 0.9723, "test_loss": 0.05007862377166748, "wall_s": 895.3023679256439}} \ No newline at end of file diff --git a/results/ep_original_v2_mnist_ep_w500_d1_s1.json b/results/ep_original_v2_mnist_ep_w500_d1_s1.json new file mode 100644 index 0000000..0a04940 --- /dev/null +++ b/results/ep_original_v2_mnist_ep_w500_d1_s1.json @@ -0,0 +1 @@ +{"args": {"method": "ep", "dataset": "mnist", "depth": 1, "width": 500, "epochs": 25, "batch_size": 20, "train_examples": 50000, "max_batches": 0, "eta": null, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 1, "device": "cuda", "outdir": "results", "tag": "ep_original_v2_mnist_ep_w500_d1_s1"}, "resolved_protocol": {"beta": 0.5, "free_steps": 20, "nudge_steps": 4, "canonical_depth": true, "learning_rates": [0.1, 0.05], "persistent_particles": true, "train_examples": 50000}, "method_source": {"paper": "https://www.frontiersin.org/journals/computational-neuroscience/articles/10.3389/fncom.2017.00024/full", "code": "https://github.com/bscellier/Towards-a-Biologically-Plausible-Backprop", "protocol": "two-phase energy-gradient dynamics; persistent free particles; random beta sign"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.955, "test_loss": 0.07901275215148926, "train_loss": 0.01013753842562437}, {"epoch_end": 1, "test_acc": 0.9645, "test_loss": 0.06253083763122559, "train_loss": 0.005079905968159437}, {"epoch_end": 2, "test_acc": 0.966, "test_loss": 0.058018816566467284, "train_loss": 0.0012329824967309833}, {"epoch_end": 3, "test_acc": 0.9684, "test_loss": 0.05564236717224121, "train_loss": 0.0008007634314708412}, {"epoch_end": 4, "test_acc": 0.9657, "test_loss": 0.059846015548706055, "train_loss": 0.0004937855992466211}, {"epoch_end": 5, "test_acc": 0.9684, "test_loss": 0.05555516471862793, "train_loss": 0.0007326625636778772}, {"epoch_end": 6, "test_acc": 0.9702, "test_loss": 0.05117148704528809, "train_loss": 0.00024014394148252904}, {"epoch_end": 7, "test_acc": 0.9687, "test_loss": 0.054611382865905764, "train_loss": 4.7521469241473824e-05}, {"epoch_end": 8, "test_acc": 0.9627, "test_loss": 0.06397499732971192, "train_loss": 0.000436745845945552}, {"epoch_end": 9, "test_acc": 0.9685, "test_loss": 0.05292844734191895, "train_loss": 6.424915568459255e-07}, {"epoch_end": 10, "test_acc": 0.9688, "test_loss": 0.052844040870666506, "train_loss": 1.239909437344977e-07}, {"epoch_end": 11, "test_acc": 0.9653, "test_loss": 0.05899420146942139, "train_loss": 0.0}, {"epoch_end": 12, "test_acc": 0.9667, "test_loss": 0.05708587989807129, "train_loss": 0.0}, {"epoch_end": 13, "test_acc": 0.965, "test_loss": 0.059490954399108883, "train_loss": 0.0}, {"epoch_end": 14, "test_acc": 0.9633, "test_loss": 0.0638974765777588, "train_loss": 0.0}, {"epoch_end": 15, "test_acc": 0.9681, "test_loss": 0.05465109443664551, "train_loss": 0.0005773885641247034}, {"epoch_end": 16, "test_acc": 0.9667, "test_loss": 0.05632556095123291, "train_loss": 0.0008614619146101177}, {"epoch_end": 17, "test_acc": 0.9671, "test_loss": 0.05757739877700806, "train_loss": 0.0}, {"epoch_end": 18, "test_acc": 0.9625, "test_loss": 0.06238394584655762, "train_loss": 0.0}, {"epoch_end": 19, "test_acc": 0.9675, "test_loss": 0.05585011367797851, "train_loss": 0.0}, {"epoch_end": 20, "test_acc": 0.9687, "test_loss": 0.053836871910095216, "train_loss": 0.0}, {"epoch_end": 21, "test_acc": 0.9689, "test_loss": 0.051564078330993655, "train_loss": 0.0}, {"epoch_end": 22, "test_acc": 0.967, "test_loss": 0.054196818923950195, "train_loss": 0.0}, {"epoch_end": 23, "test_acc": 0.9673, "test_loss": 0.05569706926345825, "train_loss": 0.0}, {"epoch_end": 24, "test_acc": 0.9685, "test_loss": 0.051756802940368656, "train_loss": 0.0}], "final": {"test_acc": 0.9685, "test_loss": 0.051756802940368656, "wall_s": 784.3743386268616}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d30_s0.json b/results/fa_scale_v2_cifar10_fa_w64_d30_s0.json new file mode 100644 index 0000000..5af3bf5 --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d30_s0.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 30, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 0, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d30_s0"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3342, "test_loss": 1.8874280395507812, "train_loss": 1.9046052694320679, "cos_fa_negg": [0.4909452795982361, 0.4232390522956848, 0.40220117568969727, 0.38053926825523376, 0.39696210622787476, 0.3763437271118164, 0.39458632469177246, 0.37708237767219543, 0.3827553391456604, 0.3844324052333832, 0.3330151438713074, 0.3514268100261688, 0.3549768626689911, 0.3381927013397217, 0.3469671905040741, 0.33116835355758667, 0.3020341992378235, 0.3173680901527405, 0.29584819078445435, 0.3117028772830963, 0.3046067953109741, 0.27343863248825073, 0.2980014979839325, 0.2900099754333496, 0.31058505177497864, 0.2815680503845215, 0.2938365638256073, 0.29301923513412476, 0.2988560199737549, 0.2993921637535095]}, {"epoch_end": 1, "test_acc": 0.3447, "test_loss": 1.8784762084960938, "train_loss": 1.8714990615844727, "cos_fa_negg": [0.50819331407547, 0.5166217088699341, 0.4913097321987152, 0.49125316739082336, 0.4961889088153839, 0.4746783971786499, 0.4780402183532715, 0.4635393023490906, 0.47434884309768677, 0.4674232304096222, 0.4387846291065216, 0.44849032163619995, 0.45123082399368286, 0.4364655017852783, 0.44448238611221313, 0.4457794725894928, 0.44029104709625244, 0.45300978422164917, 0.44533586502075195, 0.4304192066192627, 0.40630704164505005, 0.386390745639801, 0.42427414655685425, 0.3856293559074402, 0.38561591506004333, 0.410695344209671, 0.41430437564849854, 0.3892138600349426, 0.4005376696586609, 0.41247156262397766]}, {"epoch_end": 2, "test_acc": 0.3568, "test_loss": 1.834733935546875, "train_loss": 1.9247595071792603, "cos_fa_negg": [0.49743181467056274, 0.5329564213752747, 0.5088294744491577, 0.5192805528640747, 0.5168434381484985, 0.49765461683273315, 0.506064236164093, 0.5020463466644287, 0.5159478783607483, 0.5093803405761719, 0.4885573387145996, 0.49219250679016113, 0.5067838430404663, 0.4895366132259369, 0.47726115584373474, 0.4775928556919098, 0.4701252579689026, 0.48262932896614075, 0.44827818870544434, 0.4560881555080414, 0.46565869450569153, 0.450237512588501, 0.4513418674468994, 0.45273810625076294, 0.4428519606590271, 0.45138436555862427, 0.44826677441596985, 0.4268227815628052, 0.4481510519981384, 0.4620015025138855]}, {"epoch_end": 3, "test_acc": 0.3747, "test_loss": 1.8052677612304688, "train_loss": 1.7121856212615967, "cos_fa_negg": [0.579709529876709, 0.5860463380813599, 0.5753939151763916, 0.5816953182220459, 0.5704218149185181, 0.5692507028579712, 0.5684423446655273, 0.5619672536849976, 0.5667442083358765, 0.5614182949066162, 0.5500373840332031, 0.5417401790618896, 0.5395127534866333, 0.540142297744751, 0.5419105291366577, 0.5376484394073486, 0.5288273692131042, 0.5544345378875732, 0.5203129053115845, 0.5281268954277039, 0.5276598930358887, 0.5138851404190063, 0.517380952835083, 0.5202401280403137, 0.5163064002990723, 0.5297417640686035, 0.5252988338470459, 0.502061128616333, 0.5162725448608398, 0.5411205291748047]}, {"epoch_end": 4, "test_acc": 0.3761, "test_loss": 1.80203974609375, "train_loss": 1.8535563945770264, "cos_fa_negg": [0.5892921090126038, 0.6235163807868958, 0.6159988641738892, 0.6225879788398743, 0.6181880235671997, 0.6052900552749634, 0.609462559223175, 0.6113673448562622, 0.612798810005188, 0.6165719032287598, 0.6025394201278687, 0.608207106590271, 0.5959622859954834, 0.6113330125808716, 0.5956776142120361, 0.59241783618927, 0.6053488850593567, 0.620844841003418, 0.5945259928703308, 0.6040456891059875, 0.6016722917556763, 0.5936439633369446, 0.5977240800857544, 0.5856672525405884, 0.5785460472106934, 0.5821727514266968, 0.5833908915519714, 0.5552667379379272, 0.5726286172866821, 0.5835205912590027]}], "final": {"test_acc": 0.3761, "test_loss": 1.80203974609375, "wall_s": 25.621788024902344, "cos_fa_negg": [0.5892921090126038, 0.6235163807868958, 0.6159988641738892, 0.6225879788398743, 0.6181880235671997, 0.6052900552749634, 0.609462559223175, 0.6113673448562622, 0.612798810005188, 0.6165719032287598, 0.6025394201278687, 0.608207106590271, 0.5959622859954834, 0.6113330125808716, 0.5956776142120361, 0.59241783618927, 0.6053488850593567, 0.620844841003418, 0.5945259928703308, 0.6040456891059875, 0.6016722917556763, 0.5936439633369446, 0.5977240800857544, 0.5856672525405884, 0.5785460472106934, 0.5821727514266968, 0.5833908915519714, 0.5552667379379272, 0.5726286172866821, 0.5835205912590027], "loss": 1.7802929878234863}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d30_s1.json b/results/fa_scale_v2_cifar10_fa_w64_d30_s1.json new file mode 100644 index 0000000..506eae2 --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d30_s1.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 30, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 1, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d30_s1"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3316, "test_loss": 1.9141797485351562, "train_loss": 1.909462332725525, "cos_fa_negg": [0.4766767621040344, 0.46940356492996216, 0.4658898711204529, 0.463259220123291, 0.4686715602874756, 0.4795825481414795, 0.4418402910232544, 0.4256480038166046, 0.43237608671188354, 0.4125264585018158, 0.42263275384902954, 0.42192184925079346, 0.3923310935497284, 0.3610675036907196, 0.3622701168060303, 0.3606058657169342, 0.37868690490722656, 0.37752383947372437, 0.36491620540618896, 0.3429092764854431, 0.3493986129760742, 0.36007651686668396, 0.3415355086326599, 0.3417968451976776, 0.3468305170536041, 0.34194034337997437, 0.3330402970314026, 0.3199993968009949, 0.31312650442123413, 0.3317967355251312]}, {"epoch_end": 1, "test_acc": 0.3648, "test_loss": 1.8278525512695312, "train_loss": 1.8296983242034912, "cos_fa_negg": [0.5319972038269043, 0.5389089584350586, 0.5435370206832886, 0.5376653075218201, 0.546789824962616, 0.5401828289031982, 0.520318865776062, 0.5080517530441284, 0.5025713443756104, 0.4859846830368042, 0.4964368939399719, 0.48192155361175537, 0.4734562039375305, 0.44807112216949463, 0.47930634021759033, 0.4592658579349518, 0.4873005449771881, 0.45718806982040405, 0.46097421646118164, 0.44537651538848877, 0.463547945022583, 0.46133124828338623, 0.4514022469520569, 0.43518033623695374, 0.44571569561958313, 0.43578052520751953, 0.4402308166027069, 0.43494999408721924, 0.4192656874656677, 0.42841434478759766]}, {"epoch_end": 2, "test_acc": 0.3663, "test_loss": 1.8270438110351563, "train_loss": 1.879701018333435, "cos_fa_negg": [0.5650622844696045, 0.565576434135437, 0.5785539150238037, 0.5721728801727295, 0.5542012453079224, 0.553023636341095, 0.5466798543930054, 0.5340300798416138, 0.5312860012054443, 0.5238196849822998, 0.5263739824295044, 0.5156461596488953, 0.5198017358779907, 0.51140958070755, 0.5236905813217163, 0.5190930962562561, 0.5122395753860474, 0.522681474685669, 0.522364616394043, 0.5109097957611084, 0.5254349708557129, 0.5296446084976196, 0.5276458263397217, 0.5016553401947021, 0.5202258229255676, 0.5152300000190735, 0.5173885822296143, 0.5185317397117615, 0.520302414894104, 0.5193288326263428]}, {"epoch_end": 3, "test_acc": 0.3861, "test_loss": 1.79345869140625, "train_loss": 1.6362102031707764, "cos_fa_negg": [0.581800103187561, 0.5976759195327759, 0.5900301933288574, 0.5951757431030273, 0.5914549827575684, 0.5818943977355957, 0.5795469284057617, 0.5754437446594238, 0.5693224668502808, 0.5533724427223206, 0.5548047423362732, 0.5552384257316589, 0.538526713848114, 0.5285570621490479, 0.5386442542076111, 0.5361975431442261, 0.5495831966400146, 0.5351501703262329, 0.5451276898384094, 0.5406442284584045, 0.5481300354003906, 0.5381314754486084, 0.5490946173667908, 0.5306671857833862, 0.5386003255844116, 0.5177560448646545, 0.5317248702049255, 0.5189740657806396, 0.530875563621521, 0.5178715586662292]}, {"epoch_end": 4, "test_acc": 0.3874, "test_loss": 1.7648971313476562, "train_loss": 1.8415944576263428, "cos_fa_negg": [0.5855481624603271, 0.6077606081962585, 0.5960338115692139, 0.598849892616272, 0.5987787842750549, 0.5875180959701538, 0.5978735685348511, 0.5931650400161743, 0.573418378829956, 0.5684139728546143, 0.5561122298240662, 0.5580655336380005, 0.5594645738601685, 0.5398756861686707, 0.5596716403961182, 0.5640764236450195, 0.591381311416626, 0.5772109627723694, 0.5682837963104248, 0.5690616369247437, 0.579123854637146, 0.5781664848327637, 0.5688639283180237, 0.5433177947998047, 0.5620559453964233, 0.551741361618042, 0.5671685934066772, 0.5584239959716797, 0.552141547203064, 0.563515305519104]}], "final": {"test_acc": 0.3874, "test_loss": 1.7648971313476562, "wall_s": 25.781265020370483, "cos_fa_negg": [0.5855481624603271, 0.6077606081962585, 0.5960338115692139, 0.598849892616272, 0.5987787842750549, 0.5875180959701538, 0.5978735685348511, 0.5931650400161743, 0.573418378829956, 0.5684139728546143, 0.5561122298240662, 0.5580655336380005, 0.5594645738601685, 0.5398756861686707, 0.5596716403961182, 0.5640764236450195, 0.591381311416626, 0.5772109627723694, 0.5682837963104248, 0.5690616369247437, 0.579123854637146, 0.5781664848327637, 0.5688639283180237, 0.5433177947998047, 0.5620559453964233, 0.551741361618042, 0.5671685934066772, 0.5584239959716797, 0.552141547203064, 0.563515305519104], "loss": 1.7946875095367432}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d30_s2.json b/results/fa_scale_v2_cifar10_fa_w64_d30_s2.json new file mode 100644 index 0000000..708d325 --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d30_s2.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 30, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 2, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d30_s2"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3502, "test_loss": 1.8632313354492187, "train_loss": 1.950326919555664, "cos_fa_negg": [0.4240458011627197, 0.3977111577987671, 0.38496845960617065, 0.4059197008609772, 0.3933277726173401, 0.34999093413352966, 0.35084545612335205, 0.35156893730163574, 0.3405769467353821, 0.3358570337295532, 0.3309953808784485, 0.34169802069664, 0.32743945717811584, 0.3383491337299347, 0.3410959839820862, 0.32634520530700684, 0.3276805281639099, 0.33679041266441345, 0.3261553645133972, 0.34101593494415283, 0.33795565366744995, 0.32427453994750977, 0.3134707808494568, 0.3176116943359375, 0.30985379219055176, 0.309181272983551, 0.29642218351364136, 0.3076708912849426, 0.3017372488975525, 0.30471378564834595]}, {"epoch_end": 1, "test_acc": 0.3466, "test_loss": 1.8698973876953124, "train_loss": 1.8450952768325806, "cos_fa_negg": [0.5031113624572754, 0.5298810005187988, 0.5306612849235535, 0.516035795211792, 0.49602001905441284, 0.47195667028427124, 0.4910922944545746, 0.48467740416526794, 0.4783472716808319, 0.469845712184906, 0.4683968722820282, 0.45313626527786255, 0.46750614047050476, 0.4579275846481323, 0.44434666633605957, 0.45150768756866455, 0.4493997097015381, 0.45030999183654785, 0.4520753026008606, 0.45265471935272217, 0.47004544734954834, 0.43819940090179443, 0.43595200777053833, 0.4327704906463623, 0.4347604513168335, 0.4168857932090759, 0.42431098222732544, 0.41570866107940674, 0.4102857708930969, 0.4510406255722046]}, {"epoch_end": 2, "test_acc": 0.3476, "test_loss": 1.865405859375, "train_loss": 1.7917602062225342, "cos_fa_negg": [0.5044389963150024, 0.4873223602771759, 0.487909197807312, 0.4734182357788086, 0.4652928113937378, 0.4474084973335266, 0.4541834890842438, 0.46343374252319336, 0.43743568658828735, 0.4360804557800293, 0.43370985984802246, 0.4292411506175995, 0.44453179836273193, 0.42876631021499634, 0.43835678696632385, 0.44039133191108704, 0.4378235936164856, 0.4378514289855957, 0.4486132860183716, 0.43493902683258057, 0.4543517231941223, 0.44897550344467163, 0.4522092938423157, 0.43076521158218384, 0.42324554920196533, 0.3919927477836609, 0.44853323698043823, 0.41402727365493774, 0.41120249032974243, 0.4275374412536621]}, {"epoch_end": 3, "test_acc": 0.3819, "test_loss": 1.7957172119140625, "train_loss": 1.7026780843734741, "cos_fa_negg": [0.5523576736450195, 0.5548592209815979, 0.5666717290878296, 0.562723696231842, 0.5563897490501404, 0.552516758441925, 0.5463669300079346, 0.5534408092498779, 0.5344465970993042, 0.5184570550918579, 0.5257173776626587, 0.5130177736282349, 0.528563916683197, 0.5301801562309265, 0.5215397477149963, 0.5219439268112183, 0.5415947437286377, 0.5234269499778748, 0.5287728309631348, 0.5261427164077759, 0.5492882132530212, 0.5148613452911377, 0.5161013603210449, 0.5250758528709412, 0.5023256540298462, 0.4894789457321167, 0.5117560625076294, 0.4914405643939972, 0.4975869655609131, 0.5016378164291382]}, {"epoch_end": 4, "test_acc": 0.3833, "test_loss": 1.78280322265625, "train_loss": 1.7658056020736694, "cos_fa_negg": [0.569114625453949, 0.6088447570800781, 0.6146378517150879, 0.6072269678115845, 0.6054257154464722, 0.603683590888977, 0.5959355235099792, 0.5999228954315186, 0.5981593132019043, 0.5804201364517212, 0.5868228673934937, 0.5800783634185791, 0.5872102975845337, 0.5868858098983765, 0.5911449193954468, 0.5844857692718506, 0.5942168235778809, 0.5773187279701233, 0.5705900192260742, 0.5886563658714294, 0.5877513885498047, 0.5835996866226196, 0.5513861179351807, 0.5723548531532288, 0.5603209733963013, 0.557844877243042, 0.5698597431182861, 0.5543977618217468, 0.5589858293533325, 0.554373562335968]}], "final": {"test_acc": 0.3833, "test_loss": 1.78280322265625, "wall_s": 27.12492275238037, "cos_fa_negg": [0.569114625453949, 0.6088447570800781, 0.6146378517150879, 0.6072269678115845, 0.6054257154464722, 0.603683590888977, 0.5959355235099792, 0.5999228954315186, 0.5981593132019043, 0.5804201364517212, 0.5868228673934937, 0.5800783634185791, 0.5872102975845337, 0.5868858098983765, 0.5911449193954468, 0.5844857692718506, 0.5942168235778809, 0.5773187279701233, 0.5705900192260742, 0.5886563658714294, 0.5877513885498047, 0.5835996866226196, 0.5513861179351807, 0.5723548531532288, 0.5603209733963013, 0.557844877243042, 0.5698597431182861, 0.5543977618217468, 0.5589858293533325, 0.554373562335968], "loss": 1.8165271282196045}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d30_s3.json b/results/fa_scale_v2_cifar10_fa_w64_d30_s3.json new file mode 100644 index 0000000..a0dffe3 --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d30_s3.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 30, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 3, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d30_s3"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.339, "test_loss": 1.8851545532226563, "train_loss": 1.8822259902954102, "cos_fa_negg": [0.42513713240623474, 0.38999736309051514, 0.3596764802932739, 0.37136003375053406, 0.34965959191322327, 0.3440188467502594, 0.3349524140357971, 0.3297094404697418, 0.33692237734794617, 0.3094968795776367, 0.3219259977340698, 0.29217860102653503, 0.30813878774642944, 0.2871021330356598, 0.31264474987983704, 0.27388694882392883, 0.27083760499954224, 0.2660243511199951, 0.2894097566604614, 0.26757264137268066, 0.2645462453365326, 0.2562705874443054, 0.2513217031955719, 0.2775954604148865, 0.2668644189834595, 0.2451467514038086, 0.2775188982486725, 0.2696828246116638, 0.2491060495376587, 0.26449257135391235]}, {"epoch_end": 1, "test_acc": 0.3507, "test_loss": 1.8410773071289062, "train_loss": 1.8560569286346436, "cos_fa_negg": [0.4765194058418274, 0.449235737323761, 0.42133212089538574, 0.4208884835243225, 0.4256449341773987, 0.40020638704299927, 0.4088268280029297, 0.4204334616661072, 0.4032711982727051, 0.41162508726119995, 0.4092821180820465, 0.36816972494125366, 0.3946911692619324, 0.3661036193370819, 0.3956451117992401, 0.3723652958869934, 0.36759698390960693, 0.347835898399353, 0.3733481168746948, 0.369132399559021, 0.3834843635559082, 0.38170111179351807, 0.3816699981689453, 0.3787584900856018, 0.3705599904060364, 0.4003303050994873, 0.39331597089767456, 0.40656203031539917, 0.3725329637527466, 0.39084142446517944]}, {"epoch_end": 2, "test_acc": 0.3584, "test_loss": 1.8495978515625, "train_loss": 1.8948198556900024, "cos_fa_negg": [0.5376198887825012, 0.5236759185791016, 0.5078796744346619, 0.4906350374221802, 0.4941539168357849, 0.4784596562385559, 0.48308998346328735, 0.4793080985546112, 0.48401767015457153, 0.4847427010536194, 0.48444581031799316, 0.4721408784389496, 0.4817342460155487, 0.474948525428772, 0.4876556396484375, 0.4817822575569153, 0.45153316855430603, 0.44267702102661133, 0.4673696756362915, 0.443167507648468, 0.4632422924041748, 0.4393828213214874, 0.46553122997283936, 0.47176486253738403, 0.47648942470550537, 0.47068655490875244, 0.46190494298934937, 0.4884182810783386, 0.4551674723625183, 0.46508675813674927]}, {"epoch_end": 3, "test_acc": 0.3626, "test_loss": 1.8256299682617187, "train_loss": 1.7423769235610962, "cos_fa_negg": [0.5742020606994629, 0.5640236139297485, 0.558806300163269, 0.5418614149093628, 0.53712397813797, 0.5262211561203003, 0.5081871151924133, 0.531700611114502, 0.5160509943962097, 0.5216243267059326, 0.5161294341087341, 0.5102526545524597, 0.510583758354187, 0.5064926147460938, 0.5272520184516907, 0.5086629390716553, 0.5136989951133728, 0.4923973083496094, 0.50771164894104, 0.49891456961631775, 0.504051685333252, 0.4878115653991699, 0.49931252002716064, 0.47622328996658325, 0.49903613328933716, 0.484167218208313, 0.4971465468406677, 0.5168876647949219, 0.49254581332206726, 0.49998581409454346]}, {"epoch_end": 4, "test_acc": 0.3849, "test_loss": 1.790675439453125, "train_loss": 1.9715389013290405, "cos_fa_negg": [0.5724054574966431, 0.5946857929229736, 0.5821412801742554, 0.5757284760475159, 0.581839919090271, 0.5668387413024902, 0.5672246217727661, 0.5705218315124512, 0.5715498924255371, 0.5679293870925903, 0.557624101638794, 0.5738222599029541, 0.5765154361724854, 0.5549193620681763, 0.5626566410064697, 0.5601063370704651, 0.5519946813583374, 0.5522816777229309, 0.5587291717529297, 0.5747476816177368, 0.5612356066703796, 0.5631564855575562, 0.5607450604438782, 0.5656013488769531, 0.5812498331069946, 0.5720499157905579, 0.5768944621086121, 0.6002975702285767, 0.5771270394325256, 0.5757820010185242]}], "final": {"test_acc": 0.3849, "test_loss": 1.790675439453125, "wall_s": 30.01699733734131, "cos_fa_negg": [0.5724054574966431, 0.5946857929229736, 0.5821412801742554, 0.5757284760475159, 0.581839919090271, 0.5668387413024902, 0.5672246217727661, 0.5705218315124512, 0.5715498924255371, 0.5679293870925903, 0.557624101638794, 0.5738222599029541, 0.5765154361724854, 0.5549193620681763, 0.5626566410064697, 0.5601063370704651, 0.5519946813583374, 0.5522816777229309, 0.5587291717529297, 0.5747476816177368, 0.5612356066703796, 0.5631564855575562, 0.5607450604438782, 0.5656013488769531, 0.5812498331069946, 0.5720499157905579, 0.5768944621086121, 0.6002975702285767, 0.5771270394325256, 0.5757820010185242], "loss": 1.8545846939086914}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d30_s4.json b/results/fa_scale_v2_cifar10_fa_w64_d30_s4.json new file mode 100644 index 0000000..50141be --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d30_s4.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 30, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 4, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d30_s4"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3396, "test_loss": 1.888619921875, "train_loss": 1.8792197704315186, "cos_fa_negg": [0.4738484025001526, 0.43231385946273804, 0.42989277839660645, 0.43321752548217773, 0.42223286628723145, 0.4349339008331299, 0.41663846373558044, 0.4047051668167114, 0.3948816955089569, 0.3973745107650757, 0.3884226381778717, 0.3820644021034241, 0.38574808835983276, 0.3683149814605713, 0.36312663555145264, 0.3586394190788269, 0.3607451319694519, 0.37132924795150757, 0.3580601215362549, 0.3844127655029297, 0.36304718255996704, 0.3337595462799072, 0.3594875931739807, 0.36377230286598206, 0.34721988439559937, 0.3524436950683594, 0.3764152228832245, 0.3818495273590088, 0.3838444948196411, 0.3763836622238159]}, {"epoch_end": 1, "test_acc": 0.3509, "test_loss": 1.85620078125, "train_loss": 1.874675989151001, "cos_fa_negg": [0.5385580658912659, 0.5376670360565186, 0.5382713079452515, 0.544151782989502, 0.5225276350975037, 0.5188923478126526, 0.4976838231086731, 0.5010864734649658, 0.5006780624389648, 0.482230007648468, 0.5011875629425049, 0.48385089635849, 0.49593040347099304, 0.4776459336280823, 0.4922087788581848, 0.48618558049201965, 0.48741215467453003, 0.4944150447845459, 0.4698142409324646, 0.4645126461982727, 0.468385249376297, 0.4533337652683258, 0.47573649883270264, 0.4589267373085022, 0.4495609998703003, 0.46491754055023193, 0.47928571701049805, 0.4798305630683899, 0.4662959575653076, 0.4760708808898926]}, {"epoch_end": 2, "test_acc": 0.3587, "test_loss": 1.8492124755859376, "train_loss": 1.8761680126190186, "cos_fa_negg": [0.5340677499771118, 0.5445166826248169, 0.5391425490379333, 0.5439318418502808, 0.5416641235351562, 0.5369202494621277, 0.5332465767860413, 0.5167236924171448, 0.4998130202293396, 0.5067301392555237, 0.5087882280349731, 0.5007187128067017, 0.49605128169059753, 0.4994116425514221, 0.5038739442825317, 0.5004045963287354, 0.48982182145118713, 0.48241209983825684, 0.4854820668697357, 0.49072757363319397, 0.4926672577857971, 0.47565963864326477, 0.47393471002578735, 0.4786396026611328, 0.44925031065940857, 0.449349045753479, 0.4802570641040802, 0.49642378091812134, 0.5002778768539429, 0.4874247908592224]}, {"epoch_end": 3, "test_acc": 0.3586, "test_loss": 1.8216365844726563, "train_loss": 1.7275930643081665, "cos_fa_negg": [0.5820386409759521, 0.5854940414428711, 0.5835282802581787, 0.5821308493614197, 0.5959003567695618, 0.5754989385604858, 0.5615509748458862, 0.5526689291000366, 0.5447532534599304, 0.5661460161209106, 0.5542716979980469, 0.5513578653335571, 0.557745635509491, 0.5535314083099365, 0.5616623163223267, 0.5733149647712708, 0.5538445711135864, 0.5513497591018677, 0.5478129982948303, 0.5491665601730347, 0.5486970543861389, 0.5467871427536011, 0.5408225655555725, 0.5384945869445801, 0.5272632837295532, 0.5377124547958374, 0.5243248343467712, 0.5456587076187134, 0.5556647777557373, 0.5405703783035278]}, {"epoch_end": 4, "test_acc": 0.3762, "test_loss": 1.7808139526367188, "train_loss": 1.9536937475204468, "cos_fa_negg": [0.5777429342269897, 0.5929767489433289, 0.5904548764228821, 0.5940835475921631, 0.5935583114624023, 0.5868256092071533, 0.5982921123504639, 0.5854113101959229, 0.5868480205535889, 0.5754259824752808, 0.5852277278900146, 0.5770748257637024, 0.5667734146118164, 0.5903566479682922, 0.5929358005523682, 0.5889279842376709, 0.5787416696548462, 0.5770376324653625, 0.5635967254638672, 0.5796545743942261, 0.5696022510528564, 0.5661047697067261, 0.5559055805206299, 0.5579375624656677, 0.5598382949829102, 0.5615957975387573, 0.5544495582580566, 0.5674855709075928, 0.5811480283737183, 0.5714433789253235]}], "final": {"test_acc": 0.3762, "test_loss": 1.7808139526367188, "wall_s": 25.74710488319397, "cos_fa_negg": [0.5777429342269897, 0.5929767489433289, 0.5904548764228821, 0.5940835475921631, 0.5935583114624023, 0.5868256092071533, 0.5982921123504639, 0.5854113101959229, 0.5868480205535889, 0.5754259824752808, 0.5852277278900146, 0.5770748257637024, 0.5667734146118164, 0.5903566479682922, 0.5929358005523682, 0.5889279842376709, 0.5787416696548462, 0.5770376324653625, 0.5635967254638672, 0.5796545743942261, 0.5696022510528564, 0.5661047697067261, 0.5559055805206299, 0.5579375624656677, 0.5598382949829102, 0.5615957975387573, 0.5544495582580566, 0.5674855709075928, 0.5811480283737183, 0.5714433789253235], "loss": 1.7888938188552856}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d60_s0.json b/results/fa_scale_v2_cifar10_fa_w64_d60_s0.json new file mode 100644 index 0000000..0e73727 --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d60_s0.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 60, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 0, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d60_s0"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3288, "test_loss": 1.8690793579101563, "train_loss": 1.8898258209228516, "cos_fa_negg": [0.4823610186576843, 0.43921324610710144, 0.4455450177192688, 0.44498035311698914, 0.43004363775253296, 0.4347233474254608, 0.4574927091598511, 0.4405841827392578, 0.4240392744541168, 0.43400973081588745, 0.43882787227630615, 0.42421194911003113, 0.43699169158935547, 0.4261325001716614, 0.42764607071876526, 0.4198639988899231, 0.4040893316268921, 0.4073064625263214, 0.39748460054397583, 0.4031910300254822, 0.4132603704929352, 0.39888861775398254, 0.40310943126678467, 0.39498913288116455, 0.38438117504119873, 0.38148194551467896, 0.3775869309902191, 0.3723202645778656, 0.37422192096710205, 0.36403071880340576, 0.3573321998119354, 0.3591834306716919, 0.357078492641449, 0.36168208718299866, 0.3292919397354126, 0.34182655811309814, 0.34416618943214417, 0.3527018427848816, 0.33056068420410156, 0.3690992593765259, 0.34540948271751404, 0.34684860706329346, 0.3466452360153198, 0.3720093071460724, 0.3543034791946411, 0.33958205580711365, 0.3565729856491089, 0.3722684979438782, 0.35881638526916504, 0.3765505254268646, 0.35374394059181213, 0.3406728506088257, 0.34480786323547363, 0.3567156195640564, 0.36322221159935, 0.37292176485061646, 0.34832698106765747, 0.35819879174232483, 0.34089744091033936, 0.33703500032424927]}, {"epoch_end": 1, "test_acc": 0.352, "test_loss": 1.855537548828125, "train_loss": 1.8152847290039062, "cos_fa_negg": [0.5321953296661377, 0.5372573137283325, 0.5315355062484741, 0.5259463787078857, 0.5243715643882751, 0.5233645439147949, 0.5231640338897705, 0.5205551981925964, 0.5146111249923706, 0.5181606411933899, 0.5116404294967651, 0.5245670080184937, 0.5214899778366089, 0.5083082914352417, 0.5035877227783203, 0.4949599504470825, 0.5092740058898926, 0.5065683126449585, 0.4952320456504822, 0.48668038845062256, 0.4826705753803253, 0.4801673889160156, 0.49211424589157104, 0.475842148065567, 0.47032278776168823, 0.47904378175735474, 0.47992628812789917, 0.4839964210987091, 0.4638198912143707, 0.4808570146560669, 0.4625883996486664, 0.45375990867614746, 0.48702892661094666, 0.4737794101238251, 0.4596349000930786, 0.45955562591552734, 0.47665902972221375, 0.4568261206150055, 0.4710555672645569, 0.46819788217544556, 0.470027357339859, 0.4755898416042328, 0.47125527262687683, 0.4612107574939728, 0.47139111161231995, 0.4668526351451874, 0.4588811993598938, 0.46773236989974976, 0.4663543105125427, 0.46938028931617737, 0.4583485424518585, 0.45061373710632324, 0.46190664172172546, 0.473145455121994, 0.4619390666484833, 0.461521178483963, 0.4455487132072449, 0.4628101587295532, 0.4585449695587158, 0.4576566517353058]}, {"epoch_end": 2, "test_acc": 0.3547, "test_loss": 1.8502237670898438, "train_loss": 1.8814747333526611, "cos_fa_negg": [0.5259183645248413, 0.5552657842636108, 0.5536394119262695, 0.5498430728912354, 0.5454748272895813, 0.539234459400177, 0.5496188402175903, 0.5387426614761353, 0.5530259609222412, 0.5449510812759399, 0.5293883681297302, 0.5449614524841309, 0.5368536710739136, 0.5350638628005981, 0.53703773021698, 0.5276102423667908, 0.5216306447982788, 0.5242408514022827, 0.5233417749404907, 0.5254262685775757, 0.5237894058227539, 0.5172607898712158, 0.5343544483184814, 0.5031626224517822, 0.5081762075424194, 0.5132939219474792, 0.5050968527793884, 0.5060415267944336, 0.49648094177246094, 0.5150277614593506, 0.5110039710998535, 0.49309200048446655, 0.5095576047897339, 0.5051331520080566, 0.4907029867172241, 0.5035518407821655, 0.49668875336647034, 0.5045901536941528, 0.49187347292900085, 0.496600866317749, 0.49854350090026855, 0.5020742416381836, 0.4929036498069763, 0.48436620831489563, 0.49063000082969666, 0.4856266975402832, 0.5183587074279785, 0.4951627552509308, 0.5068442821502686, 0.5008949041366577, 0.485798716545105, 0.4775030016899109, 0.4861205220222473, 0.496229350566864, 0.4710626006126404, 0.4802466332912445, 0.482088565826416, 0.47286391258239746, 0.48881617188453674, 0.4921649098396301]}, {"epoch_end": 3, "test_acc": 0.373, "test_loss": 1.834519091796875, "train_loss": 1.7697551250457764, "cos_fa_negg": [0.6133296489715576, 0.5836681127548218, 0.5732977390289307, 0.5701491832733154, 0.5713095664978027, 0.5725408792495728, 0.5774715542793274, 0.5750058889389038, 0.5774861574172974, 0.5661100149154663, 0.5751547813415527, 0.5811598300933838, 0.58670574426651, 0.5702888369560242, 0.5680985450744629, 0.5760103464126587, 0.5701871514320374, 0.5580888390541077, 0.5699899792671204, 0.5558649301528931, 0.5640165209770203, 0.5626261234283447, 0.553554356098175, 0.5582845211029053, 0.5379931926727295, 0.5577686429023743, 0.5648813247680664, 0.5492110252380371, 0.553758978843689, 0.5489376783370972, 0.5452089905738831, 0.5349271297454834, 0.5422258377075195, 0.5392653942108154, 0.5419694185256958, 0.5387028455734253, 0.5601098537445068, 0.5489915609359741, 0.5340370535850525, 0.5402944087982178, 0.5446239709854126, 0.536385178565979, 0.5366347432136536, 0.5506917238235474, 0.5448618531227112, 0.5452682971954346, 0.5476751327514648, 0.5437036752700806, 0.5516558885574341, 0.5586711168289185, 0.5380712747573853, 0.548771858215332, 0.5468689203262329, 0.5608116388320923, 0.5405697822570801, 0.5412606596946716, 0.552552342414856, 0.5433241128921509, 0.540050208568573, 0.5371251106262207]}, {"epoch_end": 4, "test_acc": 0.3858, "test_loss": 1.7755390869140626, "train_loss": 1.8684698343276978, "cos_fa_negg": [0.6384724378585815, 0.5889672040939331, 0.5907251238822937, 0.6022745370864868, 0.5971429347991943, 0.5983754396438599, 0.6056829690933228, 0.5919353365898132, 0.5980709791183472, 0.600554347038269, 0.5830105543136597, 0.5869160890579224, 0.5896507501602173, 0.5730280876159668, 0.572446882724762, 0.5666447877883911, 0.56927490234375, 0.5599130392074585, 0.564217209815979, 0.5636455416679382, 0.5693211555480957, 0.5636969208717346, 0.5646611452102661, 0.5615024566650391, 0.5625544190406799, 0.5578426122665405, 0.5577962398529053, 0.5610400438308716, 0.544654369354248, 0.5479583740234375, 0.5444278717041016, 0.5470367074012756, 0.5669836401939392, 0.5496798157691956, 0.5509804487228394, 0.5483946800231934, 0.5441389083862305, 0.5505532026290894, 0.5449956059455872, 0.5592752695083618, 0.5533009767532349, 0.5517096519470215, 0.5406250953674316, 0.5506448745727539, 0.5623810291290283, 0.554097056388855, 0.549028754234314, 0.5437087416648865, 0.5568485260009766, 0.54723060131073, 0.5300626754760742, 0.5464116334915161, 0.5390413403511047, 0.5563110113143921, 0.5298236608505249, 0.5576655864715576, 0.5301382541656494, 0.5437535047531128, 0.5384819507598877, 0.5374060869216919]}], "final": {"test_acc": 0.3858, "test_loss": 1.7755390869140626, "wall_s": 47.553232192993164, "cos_fa_negg": [0.6384724378585815, 0.5889672040939331, 0.5907251238822937, 0.6022745370864868, 0.5971429347991943, 0.5983754396438599, 0.6056829690933228, 0.5919353365898132, 0.5980709791183472, 0.600554347038269, 0.5830105543136597, 0.5869160890579224, 0.5896507501602173, 0.5730280876159668, 0.572446882724762, 0.5666447877883911, 0.56927490234375, 0.5599130392074585, 0.564217209815979, 0.5636455416679382, 0.5693211555480957, 0.5636969208717346, 0.5646611452102661, 0.5615024566650391, 0.5625544190406799, 0.5578426122665405, 0.5577962398529053, 0.5610400438308716, 0.544654369354248, 0.5479583740234375, 0.5444278717041016, 0.5470367074012756, 0.5669836401939392, 0.5496798157691956, 0.5509804487228394, 0.5483946800231934, 0.5441389083862305, 0.5505532026290894, 0.5449956059455872, 0.5592752695083618, 0.5533009767532349, 0.5517096519470215, 0.5406250953674316, 0.5506448745727539, 0.5623810291290283, 0.554097056388855, 0.549028754234314, 0.5437087416648865, 0.5568485260009766, 0.54723060131073, 0.5300626754760742, 0.5464116334915161, 0.5390413403511047, 0.5563110113143921, 0.5298236608505249, 0.5576655864715576, 0.5301382541656494, 0.5437535047531128, 0.5384819507598877, 0.5374060869216919], "loss": 1.7876933813095093}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d60_s1.json b/results/fa_scale_v2_cifar10_fa_w64_d60_s1.json new file mode 100644 index 0000000..6a7d6e7 --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d60_s1.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 60, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 1, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d60_s1"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3563, "test_loss": 1.8520883911132813, "train_loss": 1.8289947509765625, "cos_fa_negg": [0.4434039294719696, 0.4753493666648865, 0.45210012793540955, 0.4445710778236389, 0.4233151078224182, 0.4386281967163086, 0.43124866485595703, 0.43249717354774475, 0.4314526915550232, 0.4371172785758972, 0.4325917661190033, 0.4178103804588318, 0.41939181089401245, 0.423803448677063, 0.397824764251709, 0.41214051842689514, 0.3985990583896637, 0.42689210176467896, 0.4151626527309418, 0.40203240513801575, 0.3870171904563904, 0.4020926356315613, 0.3947827219963074, 0.3751707673072815, 0.3694879412651062, 0.37595123052597046, 0.39026445150375366, 0.3701687455177307, 0.38291841745376587, 0.3585672974586487, 0.35089704394340515, 0.3791882395744324, 0.3689890503883362, 0.38048967719078064, 0.3693152964115143, 0.36934149265289307, 0.3613399863243103, 0.35622745752334595, 0.357387900352478, 0.3580520749092102, 0.35191234946250916, 0.3580847680568695, 0.34585046768188477, 0.32468417286872864, 0.33910810947418213, 0.3201901912689209, 0.3480328321456909, 0.3221348524093628, 0.3323505222797394, 0.33236759901046753, 0.32947802543640137, 0.3224327564239502, 0.32983532547950745, 0.34193655848503113, 0.336813747882843, 0.34628164768218994, 0.3587023615837097, 0.32447731494903564, 0.35405758023262024, 0.3118651509284973]}, {"epoch_end": 1, "test_acc": 0.3559, "test_loss": 1.8490231567382813, "train_loss": 1.8657258749008179, "cos_fa_negg": [0.48381054401397705, 0.5500714778900146, 0.5460389852523804, 0.5551108121871948, 0.5354260206222534, 0.5328056812286377, 0.5295716524124146, 0.5322346687316895, 0.5290165543556213, 0.5167255997657776, 0.5086425542831421, 0.5199854373931885, 0.5107195973396301, 0.49741896986961365, 0.5053589344024658, 0.4975630044937134, 0.497802197933197, 0.49540024995803833, 0.4730953872203827, 0.4609830379486084, 0.47414618730545044, 0.47781601548194885, 0.46880215406417847, 0.45800071954727173, 0.4519025981426239, 0.4655839800834656, 0.4633220136165619, 0.4536944627761841, 0.4562842845916748, 0.44848769903182983, 0.4466165602207184, 0.4629764258861542, 0.457511305809021, 0.44852954149246216, 0.4382184147834778, 0.4467834234237671, 0.43548059463500977, 0.4278916120529175, 0.4372745156288147, 0.4383038282394409, 0.4261924624443054, 0.4264953136444092, 0.44041693210601807, 0.4273778200149536, 0.43781012296676636, 0.4226686954498291, 0.4139231741428375, 0.4205767810344696, 0.4217721223831177, 0.40639209747314453, 0.4206266403198242, 0.43629491329193115, 0.42854616045951843, 0.42777374386787415, 0.4128763675689697, 0.4251633286476135, 0.4331933259963989, 0.40865588188171387, 0.4125667214393616, 0.4116266071796417]}, {"epoch_end": 2, "test_acc": 0.3445, "test_loss": 1.8524200805664062, "train_loss": 1.9792779684066772, "cos_fa_negg": [0.4996739327907562, 0.53097003698349, 0.5344955921173096, 0.5183107852935791, 0.5092499852180481, 0.5081530213356018, 0.49029624462127686, 0.4997502565383911, 0.5006091594696045, 0.4900766611099243, 0.5078157782554626, 0.4845874309539795, 0.4907569885253906, 0.48622655868530273, 0.4930225610733032, 0.48840683698654175, 0.47538256645202637, 0.4755973815917969, 0.4797852039337158, 0.45577991008758545, 0.46482419967651367, 0.448876291513443, 0.4540831446647644, 0.4560111165046692, 0.4597434401512146, 0.46310532093048096, 0.46400463581085205, 0.4437459111213684, 0.46478480100631714, 0.4651157855987549, 0.4526134133338928, 0.4601321220397949, 0.4619312584400177, 0.45683354139328003, 0.4414973258972168, 0.43284404277801514, 0.4386705756187439, 0.4337202310562134, 0.4660190939903259, 0.43608996272087097, 0.43362724781036377, 0.4373180866241455, 0.4312027394771576, 0.4324885606765747, 0.4457886815071106, 0.43470096588134766, 0.43383583426475525, 0.43925440311431885, 0.4228940010070801, 0.42771345376968384, 0.4482797384262085, 0.4470859169960022, 0.4353817403316498, 0.44456470012664795, 0.4503561854362488, 0.4443289339542389, 0.4503544270992279, 0.4314218759536743, 0.4567021131515503, 0.43811047077178955]}, {"epoch_end": 3, "test_acc": 0.3687, "test_loss": 1.8377136962890626, "train_loss": 1.6802724599838257, "cos_fa_negg": [0.5273182392120361, 0.5831748247146606, 0.5814053416252136, 0.5909938812255859, 0.5766831636428833, 0.5725326538085938, 0.5650118589401245, 0.5727307796478271, 0.573728084564209, 0.5533271431922913, 0.5554448366165161, 0.551802396774292, 0.5543485879898071, 0.5481820106506348, 0.548470139503479, 0.5359897613525391, 0.5342036485671997, 0.5202466249465942, 0.5323349833488464, 0.5163587331771851, 0.5249276757240295, 0.5315729975700378, 0.5271809101104736, 0.5144158601760864, 0.5094597935676575, 0.5289677381515503, 0.5272536277770996, 0.4993322193622589, 0.5213626623153687, 0.5291158556938171, 0.5208658576011658, 0.5228713750839233, 0.5079677700996399, 0.5083463191986084, 0.49892565608024597, 0.49826014041900635, 0.5057436227798462, 0.4990547299385071, 0.5177788138389587, 0.5054153203964233, 0.49945658445358276, 0.504548966884613, 0.5117955207824707, 0.5034315586090088, 0.5032660961151123, 0.48641663789749146, 0.5070471167564392, 0.507249116897583, 0.499668687582016, 0.5115169286727905, 0.5181530117988586, 0.514991283416748, 0.4934315085411072, 0.5125821232795715, 0.5001150965690613, 0.5059525966644287, 0.5163255333900452, 0.5131232142448425, 0.512364387512207, 0.5091379880905151]}, {"epoch_end": 4, "test_acc": 0.3852, "test_loss": 1.7880859741210937, "train_loss": 1.695124626159668, "cos_fa_negg": [0.5954298973083496, 0.6071167588233948, 0.6085737943649292, 0.6195099949836731, 0.5870783925056458, 0.5954540967941284, 0.579441249370575, 0.5891669988632202, 0.6023783683776855, 0.571958065032959, 0.5934351086616516, 0.5867049694061279, 0.5944731831550598, 0.5719439387321472, 0.5761458873748779, 0.5863386392593384, 0.5705896019935608, 0.5748977661132812, 0.5805848836898804, 0.5690311193466187, 0.569037914276123, 0.5742883682250977, 0.5670415759086609, 0.5568866729736328, 0.5478408932685852, 0.5524090528488159, 0.5710761547088623, 0.5617936849594116, 0.5614355802536011, 0.5690361261367798, 0.5557476878166199, 0.5640609264373779, 0.5584446787834167, 0.5709996223449707, 0.5520274639129639, 0.5424160361289978, 0.5445843935012817, 0.5429826974868774, 0.5632505416870117, 0.546127200126648, 0.5614889860153198, 0.5613453388214111, 0.5716320872306824, 0.5576316118240356, 0.5690866112709045, 0.5516164302825928, 0.5532021522521973, 0.5467175245285034, 0.5490179657936096, 0.5437058806419373, 0.5568556785583496, 0.5495596528053284, 0.5270772576332092, 0.5407581329345703, 0.5481576919555664, 0.5502917170524597, 0.5362732410430908, 0.5447664260864258, 0.5527576208114624, 0.5413040518760681]}], "final": {"test_acc": 0.3852, "test_loss": 1.7880859741210937, "wall_s": 52.45456576347351, "cos_fa_negg": [0.5954298973083496, 0.6071167588233948, 0.6085737943649292, 0.6195099949836731, 0.5870783925056458, 0.5954540967941284, 0.579441249370575, 0.5891669988632202, 0.6023783683776855, 0.571958065032959, 0.5934351086616516, 0.5867049694061279, 0.5944731831550598, 0.5719439387321472, 0.5761458873748779, 0.5863386392593384, 0.5705896019935608, 0.5748977661132812, 0.5805848836898804, 0.5690311193466187, 0.569037914276123, 0.5742883682250977, 0.5670415759086609, 0.5568866729736328, 0.5478408932685852, 0.5524090528488159, 0.5710761547088623, 0.5617936849594116, 0.5614355802536011, 0.5690361261367798, 0.5557476878166199, 0.5640609264373779, 0.5584446787834167, 0.5709996223449707, 0.5520274639129639, 0.5424160361289978, 0.5445843935012817, 0.5429826974868774, 0.5632505416870117, 0.546127200126648, 0.5614889860153198, 0.5613453388214111, 0.5716320872306824, 0.5576316118240356, 0.5690866112709045, 0.5516164302825928, 0.5532021522521973, 0.5467175245285034, 0.5490179657936096, 0.5437058806419373, 0.5568556785583496, 0.5495596528053284, 0.5270772576332092, 0.5407581329345703, 0.5481576919555664, 0.5502917170524597, 0.5362732410430908, 0.5447664260864258, 0.5527576208114624, 0.5413040518760681], "loss": 1.8258771896362305}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d60_s2.json b/results/fa_scale_v2_cifar10_fa_w64_d60_s2.json new file mode 100644 index 0000000..da58c0e --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d60_s2.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 60, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 2, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d60_s2"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3359, "test_loss": 1.8631806884765625, "train_loss": 1.9644216299057007, "cos_fa_negg": [0.44939544796943665, 0.4366541802883148, 0.43394482135772705, 0.4342002272605896, 0.42056506872177124, 0.42718034982681274, 0.41029632091522217, 0.4167521893978119, 0.41635602712631226, 0.4175673723220825, 0.40919366478919983, 0.4104008078575134, 0.4157550632953644, 0.4037930369377136, 0.391323983669281, 0.4014747738838196, 0.398845374584198, 0.379844069480896, 0.39526769518852234, 0.39278116822242737, 0.37879711389541626, 0.37686657905578613, 0.3717030882835388, 0.3726053237915039, 0.380940705537796, 0.38014256954193115, 0.3566117584705353, 0.3757815957069397, 0.3652135133743286, 0.37618380784988403, 0.37987345457077026, 0.3654419779777527, 0.3732888698577881, 0.3497534394264221, 0.35675305128097534, 0.35354965925216675, 0.36713194847106934, 0.3522709012031555, 0.3643171787261963, 0.35341355204582214, 0.35938313603401184, 0.34984090924263, 0.3531340956687927, 0.34750843048095703, 0.3363109230995178, 0.3341760039329529, 0.34449097514152527, 0.31935036182403564, 0.3346045911312103, 0.320520281791687, 0.34237152338027954, 0.32223987579345703, 0.306252121925354, 0.3292592167854309, 0.31832265853881836, 0.32364416122436523, 0.3196065425872803, 0.3068355321884155, 0.33528730273246765, 0.3165544271469116]}, {"epoch_end": 1, "test_acc": 0.3423, "test_loss": 1.8771573974609375, "train_loss": 1.8068002462387085, "cos_fa_negg": [0.5989673137664795, 0.5481294393539429, 0.5504796504974365, 0.5510965585708618, 0.5440082550048828, 0.5441622734069824, 0.5243369340896606, 0.5315028429031372, 0.5257682800292969, 0.5433398485183716, 0.521862268447876, 0.5208358764648438, 0.5234832763671875, 0.5234793424606323, 0.5200520753860474, 0.5239675045013428, 0.5011940598487854, 0.49975112080574036, 0.5130802392959595, 0.49904340505599976, 0.48761796951293945, 0.5097000598907471, 0.4896298944950104, 0.46762827038764954, 0.4842488765716553, 0.48649731278419495, 0.46843239665031433, 0.4813463091850281, 0.45167070627212524, 0.4809161424636841, 0.4774864912033081, 0.4785611927509308, 0.4806307554244995, 0.46728670597076416, 0.47586727142333984, 0.4856345057487488, 0.49004948139190674, 0.46034395694732666, 0.4814615845680237, 0.47943562269210815, 0.46442943811416626, 0.469482421875, 0.4774102568626404, 0.456780344247818, 0.4740329384803772, 0.4752022922039032, 0.48966607451438904, 0.461667001247406, 0.4651767909526825, 0.4448210597038269, 0.4727221131324768, 0.4699898958206177, 0.458011656999588, 0.4534984529018402, 0.4578160345554352, 0.45996418595314026, 0.4655182361602783, 0.46504542231559753, 0.47580963373184204, 0.47343847155570984]}, {"epoch_end": 2, "test_acc": 0.3694, "test_loss": 1.8243054443359374, "train_loss": 1.9807122945785522, "cos_fa_negg": [0.5286617279052734, 0.5457892417907715, 0.5506066083908081, 0.5545300245285034, 0.5410588979721069, 0.544946551322937, 0.5355464816093445, 0.5444557070732117, 0.5286744832992554, 0.5439136028289795, 0.5290207266807556, 0.5215548872947693, 0.5267431139945984, 0.5218671560287476, 0.5272413492202759, 0.5291169285774231, 0.5167948007583618, 0.5182696580886841, 0.5305798053741455, 0.5261926651000977, 0.5144145488739014, 0.5108032822608948, 0.5003995895385742, 0.4954881966114044, 0.5009925365447998, 0.4865628480911255, 0.4836794137954712, 0.5059030652046204, 0.49152591824531555, 0.4917759299278259, 0.4948107600212097, 0.4895933270454407, 0.47700774669647217, 0.49554768204689026, 0.5023249387741089, 0.4854092001914978, 0.483333021402359, 0.4671231508255005, 0.46723711490631104, 0.48210036754608154, 0.4798857569694519, 0.4726146459579468, 0.4611623287200928, 0.47149208188056946, 0.4867120385169983, 0.4778882563114166, 0.47793519496917725, 0.4604351818561554, 0.47040843963623047, 0.4611439108848572, 0.47657862305641174, 0.49016380310058594, 0.4612923264503479, 0.4464431405067444, 0.4613121747970581, 0.4720686078071594, 0.45077893137931824, 0.4593217372894287, 0.4726870357990265, 0.4707539677619934]}, {"epoch_end": 3, "test_acc": 0.3734, "test_loss": 1.8144768920898438, "train_loss": 1.7867228984832764, "cos_fa_negg": [0.5602981448173523, 0.5777850151062012, 0.5710012316703796, 0.5713973045349121, 0.5705352425575256, 0.5813796520233154, 0.5614778995513916, 0.5658378601074219, 0.5631442070007324, 0.5697122812271118, 0.5618638396263123, 0.5601681470870972, 0.5555503964424133, 0.5635559558868408, 0.5548182725906372, 0.5579586029052734, 0.5570316910743713, 0.5511083006858826, 0.5582812428474426, 0.5486480593681335, 0.5291706323623657, 0.5516136884689331, 0.5429633259773254, 0.5429675579071045, 0.5331754088401794, 0.538835883140564, 0.5308257341384888, 0.5312422513961792, 0.5334263443946838, 0.5362647771835327, 0.5307689309120178, 0.5209136605262756, 0.5426049828529358, 0.5342735648155212, 0.5423767566680908, 0.5251208543777466, 0.5270175933837891, 0.5384640693664551, 0.5252139568328857, 0.5293442010879517, 0.538006067276001, 0.5293540358543396, 0.5252540111541748, 0.5201995372772217, 0.5285300612449646, 0.5403411388397217, 0.5258945822715759, 0.5231702327728271, 0.5021091103553772, 0.5154520273208618, 0.5192345380783081, 0.5404201149940491, 0.5206915140151978, 0.5157338380813599, 0.5283565521240234, 0.5133250951766968, 0.5164327621459961, 0.49361634254455566, 0.5081863403320312, 0.5124826431274414]}, {"epoch_end": 4, "test_acc": 0.3779, "test_loss": 1.7816672241210938, "train_loss": 1.774815320968628, "cos_fa_negg": [0.5561373233795166, 0.5685048699378967, 0.5829553008079529, 0.5837739109992981, 0.5603460073471069, 0.5739759206771851, 0.556145191192627, 0.5623235702514648, 0.5566625595092773, 0.5622374415397644, 0.5612963438034058, 0.5552580952644348, 0.5564990043640137, 0.5567346811294556, 0.5703528523445129, 0.5662370920181274, 0.5424145460128784, 0.5378351211547852, 0.5611943006515503, 0.5398880243301392, 0.5484473705291748, 0.5539804100990295, 0.5572672486305237, 0.5307990908622742, 0.5294938683509827, 0.5393015146255493, 0.5131348967552185, 0.5319948196411133, 0.5116909146308899, 0.5348865985870361, 0.5389769673347473, 0.5338073372840881, 0.5374222993850708, 0.5413222312927246, 0.539928674697876, 0.5410728454589844, 0.5417746305465698, 0.5380059480667114, 0.5413497686386108, 0.5305360555648804, 0.5299579501152039, 0.5421008467674255, 0.5402143001556396, 0.5502578020095825, 0.5416511297225952, 0.5360346436500549, 0.543950617313385, 0.5358645915985107, 0.5487207770347595, 0.5384303331375122, 0.5330339670181274, 0.5439651012420654, 0.535294771194458, 0.5267559289932251, 0.540719747543335, 0.5372808575630188, 0.5273672938346863, 0.514391303062439, 0.522276759147644, 0.525122344493866]}], "final": {"test_acc": 0.3779, "test_loss": 1.7816672241210938, "wall_s": 49.329002380371094, "cos_fa_negg": [0.5561373233795166, 0.5685048699378967, 0.5829553008079529, 0.5837739109992981, 0.5603460073471069, 0.5739759206771851, 0.556145191192627, 0.5623235702514648, 0.5566625595092773, 0.5622374415397644, 0.5612963438034058, 0.5552580952644348, 0.5564990043640137, 0.5567346811294556, 0.5703528523445129, 0.5662370920181274, 0.5424145460128784, 0.5378351211547852, 0.5611943006515503, 0.5398880243301392, 0.5484473705291748, 0.5539804100990295, 0.5572672486305237, 0.5307990908622742, 0.5294938683509827, 0.5393015146255493, 0.5131348967552185, 0.5319948196411133, 0.5116909146308899, 0.5348865985870361, 0.5389769673347473, 0.5338073372840881, 0.5374222993850708, 0.5413222312927246, 0.539928674697876, 0.5410728454589844, 0.5417746305465698, 0.5380059480667114, 0.5413497686386108, 0.5305360555648804, 0.5299579501152039, 0.5421008467674255, 0.5402143001556396, 0.5502578020095825, 0.5416511297225952, 0.5360346436500549, 0.543950617313385, 0.5358645915985107, 0.5487207770347595, 0.5384303331375122, 0.5330339670181274, 0.5439651012420654, 0.535294771194458, 0.5267559289932251, 0.540719747543335, 0.5372808575630188, 0.5273672938346863, 0.514391303062439, 0.522276759147644, 0.525122344493866], "loss": 1.7961766719818115}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d60_s3.json b/results/fa_scale_v2_cifar10_fa_w64_d60_s3.json new file mode 100644 index 0000000..c13db9f --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d60_s3.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 60, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 3, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d60_s3"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3371, "test_loss": 1.861859423828125, "train_loss": 1.8896558284759521, "cos_fa_negg": [0.46320652961730957, 0.4257885813713074, 0.4397363066673279, 0.432605117559433, 0.4112982451915741, 0.42634016275405884, 0.40807193517684937, 0.4004955291748047, 0.401429682970047, 0.3900667130947113, 0.400176465511322, 0.38530343770980835, 0.38670045137405396, 0.3957256078720093, 0.38539618253707886, 0.3679041266441345, 0.3648647367954254, 0.3543320894241333, 0.3745007812976837, 0.3677763342857361, 0.3635929822921753, 0.35941705107688904, 0.3566517233848572, 0.3473321497440338, 0.3390972912311554, 0.3280073404312134, 0.3524836599826813, 0.3490320146083832, 0.35690993070602417, 0.33472394943237305, 0.33037593960762024, 0.34541386365890503, 0.3235771059989929, 0.32440251111984253, 0.33615171909332275, 0.3400499224662781, 0.32962390780448914, 0.32591712474823, 0.3140389323234558, 0.3203296959400177, 0.33130893111228943, 0.3200613558292389, 0.3393248915672302, 0.31637459993362427, 0.3182802200317383, 0.3315473794937134, 0.315340131521225, 0.3277358114719391, 0.31976282596588135, 0.3325124979019165, 0.3139767050743103, 0.32010650634765625, 0.3104420602321625, 0.3102288246154785, 0.3333125710487366, 0.3348211646080017, 0.33296167850494385, 0.31354162096977234, 0.3374548852443695, 0.32371705770492554]}, {"epoch_end": 1, "test_acc": 0.3539, "test_loss": 1.8531835327148438, "train_loss": 1.8171195983886719, "cos_fa_negg": [0.5110608339309692, 0.49377405643463135, 0.48862284421920776, 0.48893123865127563, 0.47957271337509155, 0.48144298791885376, 0.47673094272613525, 0.47782325744628906, 0.4636135697364807, 0.46655476093292236, 0.46682631969451904, 0.473413348197937, 0.47238045930862427, 0.4423181712627411, 0.45704030990600586, 0.4450656473636627, 0.44080597162246704, 0.4409937262535095, 0.43961045145988464, 0.431679904460907, 0.4453188180923462, 0.43329206109046936, 0.44268807768821716, 0.43837714195251465, 0.44317513704299927, 0.4294191002845764, 0.4297705888748169, 0.4360629916191101, 0.4363173246383667, 0.42604148387908936, 0.4308382570743561, 0.434598445892334, 0.42274725437164307, 0.43787211179733276, 0.43153244256973267, 0.4254937171936035, 0.4210498034954071, 0.40998202562332153, 0.4096968173980713, 0.4308662712574005, 0.40012603998184204, 0.4130649268627167, 0.42151808738708496, 0.40891414880752563, 0.413642942905426, 0.4024048447608948, 0.4036053717136383, 0.40587306022644043, 0.4018712639808655, 0.4181516468524933, 0.41807782649993896, 0.41712310910224915, 0.37534186244010925, 0.41916322708129883, 0.40612488985061646, 0.4152523875236511, 0.4158405661582947, 0.4236413836479187, 0.42596733570098877, 0.38492950797080994]}, {"epoch_end": 2, "test_acc": 0.3583, "test_loss": 1.8413156860351563, "train_loss": 1.8513799905776978, "cos_fa_negg": [0.5382483601570129, 0.5500413179397583, 0.5439579486846924, 0.5453837513923645, 0.5439662337303162, 0.5354052782058716, 0.5405007600784302, 0.5361800193786621, 0.5185878276824951, 0.5249046087265015, 0.5233983397483826, 0.5127404928207397, 0.5118182301521301, 0.5046526193618774, 0.5036765336990356, 0.5090753436088562, 0.4916156232357025, 0.5042204856872559, 0.4847179651260376, 0.48309236764907837, 0.4898642301559448, 0.47619855403900146, 0.49260222911834717, 0.47661811113357544, 0.4753396213054657, 0.46548259258270264, 0.4708009660243988, 0.4807940721511841, 0.47095638513565063, 0.45772644877433777, 0.470537394285202, 0.4774637818336487, 0.4492161273956299, 0.46155989170074463, 0.46452462673187256, 0.4721144437789917, 0.45168912410736084, 0.46064117550849915, 0.45554184913635254, 0.4564598798751831, 0.4569174349308014, 0.458788126707077, 0.4428882896900177, 0.4619501829147339, 0.451640784740448, 0.44203299283981323, 0.4463663101196289, 0.43894028663635254, 0.4602811634540558, 0.4624049663543701, 0.4319654107093811, 0.441896915435791, 0.4256783723831177, 0.4452221691608429, 0.4409179985523224, 0.42288559675216675, 0.4503013491630554, 0.427615225315094, 0.45041316747665405, 0.4339671730995178]}, {"epoch_end": 3, "test_acc": 0.3705, "test_loss": 1.83051025390625, "train_loss": 1.7035131454467773, "cos_fa_negg": [0.5514177083969116, 0.5784920454025269, 0.5692345499992371, 0.5728849172592163, 0.5593810081481934, 0.5694760084152222, 0.5654473900794983, 0.5577222108840942, 0.559241771697998, 0.5639159679412842, 0.5488275289535522, 0.5537726879119873, 0.5478445291519165, 0.5584410429000854, 0.547154426574707, 0.5386309623718262, 0.5452484488487244, 0.536888599395752, 0.5351970195770264, 0.5362622141838074, 0.5367696285247803, 0.5312141180038452, 0.5303919315338135, 0.5237957239151001, 0.5282433032989502, 0.5326159000396729, 0.5218751430511475, 0.5201091766357422, 0.5168887376785278, 0.5120941996574402, 0.5161045789718628, 0.5202073454856873, 0.5222892761230469, 0.5050457119941711, 0.5188188552856445, 0.5331007242202759, 0.5190242528915405, 0.51570725440979, 0.5153571367263794, 0.5132036805152893, 0.5223312377929688, 0.5082389712333679, 0.5166890025138855, 0.5208811163902283, 0.5109480023384094, 0.5020143389701843, 0.5233063697814941, 0.49994051456451416, 0.5089154839515686, 0.5268007516860962, 0.5048105716705322, 0.5122732520103455, 0.4907379746437073, 0.5201115608215332, 0.5261105298995972, 0.5106496810913086, 0.5188218355178833, 0.523541271686554, 0.5183056592941284, 0.5048911571502686]}, {"epoch_end": 4, "test_acc": 0.3842, "test_loss": 1.787637109375, "train_loss": 1.8954216241836548, "cos_fa_negg": [0.5674128532409668, 0.5932905673980713, 0.582792341709137, 0.5905674695968628, 0.5930498838424683, 0.5913361310958862, 0.5906393527984619, 0.5785386562347412, 0.5767732858657837, 0.5734540224075317, 0.5617853403091431, 0.5816353559494019, 0.5641161203384399, 0.5730141401290894, 0.5493698120117188, 0.5571058988571167, 0.5543322563171387, 0.5596646070480347, 0.5573434829711914, 0.5422120094299316, 0.5475142002105713, 0.5555418133735657, 0.5366872549057007, 0.540016233921051, 0.540418267250061, 0.5320790410041809, 0.5235832333564758, 0.5440571904182434, 0.5368039608001709, 0.516162097454071, 0.5275815725326538, 0.5288594961166382, 0.5285940170288086, 0.5170311331748962, 0.5187094211578369, 0.536198616027832, 0.5230274200439453, 0.5289645195007324, 0.5149238109588623, 0.5166803598403931, 0.519931972026825, 0.5199311971664429, 0.5291275978088379, 0.5329493284225464, 0.5306320190429688, 0.522091269493103, 0.5172915458679199, 0.5178797245025635, 0.5167713165283203, 0.5222961902618408, 0.5230103731155396, 0.5197954773902893, 0.5136648416519165, 0.5350509881973267, 0.5336995124816895, 0.536629855632782, 0.5327650904655457, 0.5384581685066223, 0.5327661037445068, 0.5296834111213684]}], "final": {"test_acc": 0.3842, "test_loss": 1.787637109375, "wall_s": 50.567131996154785, "cos_fa_negg": [0.5674128532409668, 0.5932905673980713, 0.582792341709137, 0.5905674695968628, 0.5930498838424683, 0.5913361310958862, 0.5906393527984619, 0.5785386562347412, 0.5767732858657837, 0.5734540224075317, 0.5617853403091431, 0.5816353559494019, 0.5641161203384399, 0.5730141401290894, 0.5493698120117188, 0.5571058988571167, 0.5543322563171387, 0.5596646070480347, 0.5573434829711914, 0.5422120094299316, 0.5475142002105713, 0.5555418133735657, 0.5366872549057007, 0.540016233921051, 0.540418267250061, 0.5320790410041809, 0.5235832333564758, 0.5440571904182434, 0.5368039608001709, 0.516162097454071, 0.5275815725326538, 0.5288594961166382, 0.5285940170288086, 0.5170311331748962, 0.5187094211578369, 0.536198616027832, 0.5230274200439453, 0.5289645195007324, 0.5149238109588623, 0.5166803598403931, 0.519931972026825, 0.5199311971664429, 0.5291275978088379, 0.5329493284225464, 0.5306320190429688, 0.522091269493103, 0.5172915458679199, 0.5178797245025635, 0.5167713165283203, 0.5222961902618408, 0.5230103731155396, 0.5197954773902893, 0.5136648416519165, 0.5350509881973267, 0.5336995124816895, 0.536629855632782, 0.5327650904655457, 0.5384581685066223, 0.5327661037445068, 0.5296834111213684], "loss": 1.8596863746643066}} \ No newline at end of file diff --git a/results/fa_scale_v2_cifar10_fa_w64_d60_s4.json b/results/fa_scale_v2_cifar10_fa_w64_d60_s4.json new file mode 100644 index 0000000..1d3ff03 --- /dev/null +++ b/results/fa_scale_v2_cifar10_fa_w64_d60_s4.json @@ -0,0 +1 @@ +{"args": {"method": "fa", "dataset": "cifar10", "depth": 60, "width": 64, "epochs": 5, "batch_size": 128, "train_examples": 0, "max_batches": 0, "eta": 0.05, "momentum": 0.9, "act": "tanh", "residual": 1, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "pepita_decay_epochs": null, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": null, "ep_dt": 0.5, "ep_free_steps": null, "ep_nudge_steps": null, "ep_persistent": 1, "canonical_preprocess": 1, "probe_bs": 512, "seed": 4, "device": "cuda", "outdir": "results", "tag": "fa_scale_v2_cifar10_fa_w64_d60_s4"}, "resolved_protocol": {"eta": 0.05, "feedback_scale": 1.0}, "method_source": {"paper": "https://www.nature.com/articles/ncomms13276", "protocol": "fixed random sequential feedback matrices"}, "provenance": {"git_commit": "5b9de3f36c91ea8ad57fc4dae5aa0c04f08a7a26", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.3342, "test_loss": 1.8863754150390626, "train_loss": 1.8797471523284912, "cos_fa_negg": [0.4667429029941559, 0.42423054575920105, 0.4201713800430298, 0.4209944009780884, 0.3996551036834717, 0.41734737157821655, 0.41233718395233154, 0.40422868728637695, 0.39915502071380615, 0.40125325322151184, 0.3988555073738098, 0.4084082245826721, 0.3879321813583374, 0.37256062030792236, 0.39565426111221313, 0.3804866671562195, 0.3879329562187195, 0.38461583852767944, 0.3979330062866211, 0.37709397077560425, 0.3726082444190979, 0.3655519485473633, 0.37531542778015137, 0.38034746050834656, 0.3949185311794281, 0.3508831262588501, 0.37045541405677795, 0.36917346715927124, 0.3586103022098541, 0.3307248055934906, 0.3290661871433258, 0.3301212787628174, 0.35567307472229004, 0.3403092622756958, 0.32811859250068665, 0.336361289024353, 0.34013813734054565, 0.31454360485076904, 0.3292902708053589, 0.3264821171760559, 0.3197968602180481, 0.3375464677810669, 0.32243943214416504, 0.292025089263916, 0.31205785274505615, 0.28534984588623047, 0.31682032346725464, 0.29787302017211914, 0.304057776927948, 0.3050808906555176, 0.3103814423084259, 0.2895582914352417, 0.2897968292236328, 0.29520976543426514, 0.2949574589729309, 0.27834630012512207, 0.2683233916759491, 0.25257566571235657, 0.27641621232032776, 0.285957932472229]}, {"epoch_end": 1, "test_acc": 0.3591, "test_loss": 1.8499001953125, "train_loss": 1.7514699697494507, "cos_fa_negg": [0.5668843984603882, 0.5454338192939758, 0.536693811416626, 0.5316558480262756, 0.5098718404769897, 0.5088881850242615, 0.5154428482055664, 0.49581196904182434, 0.4963357448577881, 0.5012784600257874, 0.5056829452514648, 0.5059401988983154, 0.5127367973327637, 0.5110639929771423, 0.5179117321968079, 0.5034053325653076, 0.5156463384628296, 0.5069144368171692, 0.5106117129325867, 0.5042880177497864, 0.4940105974674225, 0.49593013525009155, 0.4862178564071655, 0.4857701063156128, 0.4858770966529846, 0.48147886991500854, 0.48575928807258606, 0.4719793200492859, 0.4929683804512024, 0.4822253882884979, 0.4689655900001526, 0.4554988145828247, 0.4627928137779236, 0.46522316336631775, 0.47390198707580566, 0.47575533390045166, 0.4694070816040039, 0.44763487577438354, 0.4580671489238739, 0.4316140413284302, 0.4358707666397095, 0.43577855825424194, 0.43419045209884644, 0.4268239736557007, 0.4193499982357025, 0.4305728077888489, 0.4428068995475769, 0.43655261397361755, 0.427792489528656, 0.4277206063270569, 0.4212293028831482, 0.4079030454158783, 0.4444389045238495, 0.4318539798259735, 0.43093910813331604, 0.4088263511657715, 0.4362547993659973, 0.4097123146057129, 0.414814829826355, 0.41251111030578613]}, {"epoch_end": 2, "test_acc": 0.3534, "test_loss": 1.8372462158203124, "train_loss": 1.8935067653656006, "cos_fa_negg": [0.5279629230499268, 0.5345975160598755, 0.5365589261054993, 0.5327020883560181, 0.5214510560035706, 0.5152796506881714, 0.5139201879501343, 0.5125000476837158, 0.5052890777587891, 0.5183135271072388, 0.5069546699523926, 0.5020056366920471, 0.496100515127182, 0.4994444251060486, 0.4942355751991272, 0.4986794888973236, 0.4849238097667694, 0.4924459457397461, 0.4886486828327179, 0.48195895552635193, 0.47640207409858704, 0.48143523931503296, 0.45834481716156006, 0.4679444432258606, 0.47565215826034546, 0.46874791383743286, 0.46468669176101685, 0.47218069434165955, 0.4781079888343811, 0.47077614068984985, 0.47566914558410645, 0.4607524871826172, 0.4719164967536926, 0.4633232355117798, 0.46662282943725586, 0.4606967568397522, 0.4595995843410492, 0.4486594796180725, 0.4513043761253357, 0.44833412766456604, 0.448646605014801, 0.44493788480758667, 0.44674092531204224, 0.4288603663444519, 0.44137415289878845, 0.4435054659843445, 0.45862704515457153, 0.43966758251190186, 0.4393945634365082, 0.4384845197200775, 0.4285985231399536, 0.4368751049041748, 0.4366716146469116, 0.4202233552932739, 0.43780380487442017, 0.4325675368309021, 0.4427779018878937, 0.4406946003437042, 0.4110482633113861, 0.42125993967056274]}, {"epoch_end": 3, "test_acc": 0.3754, "test_loss": 1.819205810546875, "train_loss": 1.7043516635894775, "cos_fa_negg": [0.5547927618026733, 0.5445938110351562, 0.547030508518219, 0.5435748100280762, 0.5415639281272888, 0.5497273206710815, 0.5327021479606628, 0.5330561399459839, 0.5317514538764954, 0.5423996448516846, 0.5529512763023376, 0.5359492897987366, 0.5314412117004395, 0.5332818627357483, 0.5341166257858276, 0.540582537651062, 0.5240219235420227, 0.5321224927902222, 0.529065728187561, 0.5148005485534668, 0.5235733985900879, 0.5143287777900696, 0.5237703323364258, 0.5109783411026001, 0.51845383644104, 0.5149438381195068, 0.5119233131408691, 0.5082097053527832, 0.5169225335121155, 0.5198193788528442, 0.5239824056625366, 0.5136945247650146, 0.5106731653213501, 0.4957577884197235, 0.5247213244438171, 0.5185501575469971, 0.49810633063316345, 0.48849132657051086, 0.5049152970314026, 0.5075912475585938, 0.5330827832221985, 0.4963948428630829, 0.5061274766921997, 0.4813365340232849, 0.485495001077652, 0.48008739948272705, 0.5169621706008911, 0.4925724267959595, 0.49086418747901917, 0.5092265605926514, 0.5012367963790894, 0.45922571420669556, 0.4788505434989929, 0.4719175696372986, 0.4928637146949768, 0.4750933051109314, 0.49555832147598267, 0.4771580100059509, 0.4757564067840576, 0.4897497892379761]}, {"epoch_end": 4, "test_acc": 0.3883, "test_loss": 1.7843109375, "train_loss": 1.8483667373657227, "cos_fa_negg": [0.5490950345993042, 0.5862316489219666, 0.5865992307662964, 0.5877746939659119, 0.5851563215255737, 0.5847951769828796, 0.5762289762496948, 0.5886124968528748, 0.5720963478088379, 0.5767730474472046, 0.5836862921714783, 0.5780237913131714, 0.5739148259162903, 0.5748844146728516, 0.5725674629211426, 0.5853974223136902, 0.5893925428390503, 0.5881185531616211, 0.5775508284568787, 0.5729501843452454, 0.584290623664856, 0.5723841190338135, 0.5640333890914917, 0.5541241765022278, 0.5794323682785034, 0.5646258592605591, 0.5573451519012451, 0.5617077350616455, 0.5657674074172974, 0.5507423877716064, 0.5698302388191223, 0.5676702260971069, 0.561833381652832, 0.5604091286659241, 0.5732583999633789, 0.5738811492919922, 0.5599881410598755, 0.5489733219146729, 0.5643110871315002, 0.5530603528022766, 0.5705825686454773, 0.5562623739242554, 0.5537739992141724, 0.5396151542663574, 0.5509980916976929, 0.5530914068222046, 0.5573710799217224, 0.5503878593444824, 0.5641669631004333, 0.5631964802742004, 0.5538578033447266, 0.534362256526947, 0.5611786842346191, 0.5410066843032837, 0.5423145294189453, 0.5375790596008301, 0.5578751564025879, 0.5661333203315735, 0.534415602684021, 0.5433106422424316]}], "final": {"test_acc": 0.3883, "test_loss": 1.7843109375, "wall_s": 49.088300466537476, "cos_fa_negg": [0.5490950345993042, 0.5862316489219666, 0.5865992307662964, 0.5877746939659119, 0.5851563215255737, 0.5847951769828796, 0.5762289762496948, 0.5886124968528748, 0.5720963478088379, 0.5767730474472046, 0.5836862921714783, 0.5780237913131714, 0.5739148259162903, 0.5748844146728516, 0.5725674629211426, 0.5853974223136902, 0.5893925428390503, 0.5881185531616211, 0.5775508284568787, 0.5729501843452454, 0.584290623664856, 0.5723841190338135, 0.5640333890914917, 0.5541241765022278, 0.5794323682785034, 0.5646258592605591, 0.5573451519012451, 0.5617077350616455, 0.5657674074172974, 0.5507423877716064, 0.5698302388191223, 0.5676702260971069, 0.561833381652832, 0.5604091286659241, 0.5732583999633789, 0.5738811492919922, 0.5599881410598755, 0.5489733219146729, 0.5643110871315002, 0.5530603528022766, 0.5705825686454773, 0.5562623739242554, 0.5537739992141724, 0.5396151542663574, 0.5509980916976929, 0.5530914068222046, 0.5573710799217224, 0.5503878593444824, 0.5641669631004333, 0.5631964802742004, 0.5538578033447266, 0.534362256526947, 0.5611786842346191, 0.5410066843032837, 0.5423145294189453, 0.5375790596008301, 0.5578751564025879, 0.5661333203315735, 0.534415602684021, 0.5433106422424316], "loss": 1.796858787536621}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_matched_s0.json b/results/nuis_ctrl_v1_rho0_matched_s0.json new file mode 100644 index 0000000..bffefd0 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_matched_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_matched_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3978137969970703, "cos_r_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_innovation_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_apical_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_Ac_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "r_norm": [4.8918256759643555, 4.819490432739258, 4.813833236694336], "ldr": {"dL_sdil": -0.007525444030761719, "dL_bp": -0.23199224472045898, "ratio": 0.03243834309991511}}, {"step": 100, "epoch": 0, "train_loss": 0.33160343766212463, "cos_r_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_innovation_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_apical_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_Ac_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "r_norm": [0.8375285267829895, 0.818266749382019, 0.8186335563659668]}, {"step": 200, "epoch": 0, "train_loss": 0.29867634177207947, "cos_r_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_innovation_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_apical_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_Ac_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "r_norm": [0.7242861390113831, 0.7113809585571289, 0.7089210152626038]}, {"step": 300, "epoch": 0, "train_loss": 0.28106310963630676, "cos_r_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_innovation_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_apical_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_Ac_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "r_norm": [0.5968752503395081, 0.5882158875465393, 0.586020827293396]}, {"step": 400, "epoch": 0, "train_loss": 0.13037653267383575, "cos_r_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_innovation_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_apical_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_Ac_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "r_norm": [0.5326493978500366, 0.5187801718711853, 0.5172048807144165]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9438, "test_loss": 0.19445072021484375}, {"step": 500, "epoch": 1, "train_loss": 0.3059273362159729, "cos_r_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_innovation_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_apical_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_Ac_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "r_norm": [0.5493181943893433, 0.5408297777175903, 0.5343528985977173], "ldr": {"dL_sdil": -0.017139852046966553, "dL_bp": -0.0144614577293396, "ratio": 1.1852091516467935}}, {"step": 600, "epoch": 1, "train_loss": 0.2938694357872009, "cos_r_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_innovation_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_apical_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_Ac_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "r_norm": [0.5047881007194519, 0.4929560720920563, 0.4911949336528778]}, {"step": 700, "epoch": 1, "train_loss": 0.0692189484834671, "cos_r_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_innovation_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_apical_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_Ac_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "r_norm": [0.4124377369880676, 0.4063720703125, 0.40228134393692017]}, {"step": 800, "epoch": 1, "train_loss": 0.10150999575853348, "cos_r_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_innovation_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_apical_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_Ac_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "r_norm": [0.41368532180786133, 0.408090204000473, 0.40460047125816345]}, {"step": 900, "epoch": 1, "train_loss": 0.08599556982517242, "cos_r_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_innovation_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_apical_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_Ac_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "r_norm": [0.35195469856262207, 0.34207263588905334, 0.34272995591163635]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9534, "test_loss": 0.15320639877319336}, {"step": 1000, "epoch": 2, "train_loss": 0.19156473875045776, "cos_r_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_innovation_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_apical_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_Ac_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "r_norm": [0.360003262758255, 0.3540140390396118, 0.3526112735271454], "ldr": {"dL_sdil": -0.009464025497436523, "dL_bp": -0.008196189999580383, "ratio": 1.1546859574901325}}, {"step": 1100, "epoch": 2, "train_loss": 0.10864664614200592, "cos_r_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_innovation_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_apical_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_Ac_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "r_norm": [0.28866949677467346, 0.2821740508079529, 0.28306305408477783]}, {"step": 1200, "epoch": 2, "train_loss": 0.031033102422952652, "cos_r_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_innovation_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_apical_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_Ac_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "r_norm": [0.30560463666915894, 0.30166518688201904, 0.29685312509536743]}, {"step": 1300, "epoch": 2, "train_loss": 0.15016421675682068, "cos_r_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_innovation_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_apical_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_Ac_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "r_norm": [0.279674768447876, 0.27491551637649536, 0.27148789167404175]}, {"step": 1400, "epoch": 2, "train_loss": 0.22877365350723267, "cos_r_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_innovation_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_apical_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_Ac_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "r_norm": [0.29575562477111816, 0.2879781126976013, 0.28471633791923523]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9592, "test_loss": 0.13688001098632813}, {"step": 1500, "epoch": 3, "train_loss": 0.042368389666080475, "cos_r_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_innovation_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_apical_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_Ac_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "r_norm": [0.3197419345378876, 0.3114323019981384, 0.3098006248474121], "ldr": {"dL_sdil": -0.008259907364845276, "dL_bp": -0.007200941443443298, "ratio": 1.1470593712945967}}, {"step": 1600, "epoch": 3, "train_loss": 0.05860002711415291, "cos_r_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_innovation_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_apical_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_Ac_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "r_norm": [0.30268725752830505, 0.29540443420410156, 0.2939055562019348]}, {"step": 1700, "epoch": 3, "train_loss": 0.08397884666919708, "cos_r_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_innovation_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_apical_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_Ac_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "r_norm": [0.29046741127967834, 0.2826988697052002, 0.27999138832092285]}, {"step": 1800, "epoch": 3, "train_loss": 0.10174758732318878, "cos_r_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_innovation_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_apical_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_Ac_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "r_norm": [0.30012351274490356, 0.29224371910095215, 0.28972357511520386]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9665, "test_loss": 0.1125657865524292}, {"step": 1900, "epoch": 4, "train_loss": 0.07242363691329956, "cos_r_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_innovation_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_apical_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_Ac_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "r_norm": [0.28996407985687256, 0.28326937556266785, 0.2812224328517914]}, {"step": 2000, "epoch": 4, "train_loss": 0.06877899169921875, "cos_r_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_innovation_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_apical_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_Ac_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "r_norm": [0.245533287525177, 0.24014459550380707, 0.23770779371261597], "ldr": {"dL_sdil": -0.006018199026584625, "dL_bp": -0.006462790071964264, "ratio": 0.9312075681820016}}, {"step": 2100, "epoch": 4, "train_loss": 0.05175407975912094, "cos_r_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_innovation_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_apical_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_Ac_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "r_norm": [0.24031473696231842, 0.23528200387954712, 0.23227009177207947]}, {"step": 2200, "epoch": 4, "train_loss": 0.04244621843099594, "cos_r_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_innovation_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_apical_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_Ac_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "r_norm": [0.2526210844516754, 0.24659496545791626, 0.24442610144615173]}, {"step": 2300, "epoch": 4, "train_loss": 0.12019313126802444, "cos_r_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_innovation_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_apical_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_Ac_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "r_norm": [0.21065500378608704, 0.20645348727703094, 0.20339635014533997]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9679, "test_loss": 0.11181112232208253}, {"step": 2400, "epoch": 5, "train_loss": 0.033473387360572815, "cos_r_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_innovation_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_apical_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_Ac_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "r_norm": [0.22075463831424713, 0.2167082130908966, 0.21338585019111633]}, {"step": 2500, "epoch": 5, "train_loss": 0.046647246927022934, "cos_r_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_innovation_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_apical_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_Ac_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "r_norm": [0.23498451709747314, 0.2304421216249466, 0.22597557306289673], "ldr": {"dL_sdil": -0.00819263607263565, "dL_bp": -0.009024456143379211, "ratio": 0.9078260166011416}}, {"step": 2600, "epoch": 5, "train_loss": 0.023400278761982918, "cos_r_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_innovation_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_apical_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_Ac_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "r_norm": [0.2576431632041931, 0.2516178488731384, 0.24756699800491333]}, {"step": 2700, "epoch": 5, "train_loss": 0.05335155129432678, "cos_r_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_innovation_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_apical_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_Ac_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "r_norm": [0.2206578552722931, 0.2165922373533249, 0.2133471816778183]}, {"step": 2800, "epoch": 5, "train_loss": 0.020739704370498657, "cos_r_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_innovation_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_apical_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_Ac_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "r_norm": [0.2329060584306717, 0.2280796319246292, 0.22505390644073486]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9685, "test_loss": 0.11429850444793702}, {"step": 2900, "epoch": 6, "train_loss": 0.02710738405585289, "cos_r_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_innovation_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_apical_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_Ac_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "r_norm": [0.2212202548980713, 0.21703368425369263, 0.21472647786140442]}, {"step": 3000, "epoch": 6, "train_loss": 0.003210586030036211, "cos_r_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_innovation_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_apical_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_Ac_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "r_norm": [0.19350673258304596, 0.1897171288728714, 0.18670257925987244], "ldr": {"dL_sdil": -0.007917709648609161, "dL_bp": -0.009828649461269379, "ratio": 0.805574527793423}}, {"step": 3100, "epoch": 6, "train_loss": 0.06387314200401306, "cos_r_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_innovation_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_apical_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_Ac_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "r_norm": [0.17050178349018097, 0.16748245060443878, 0.16528522968292236]}, {"step": 3200, "epoch": 6, "train_loss": 0.04849023371934891, "cos_r_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_innovation_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_apical_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_Ac_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "r_norm": [0.18880192935466766, 0.18442402780056, 0.18181553483009338]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9723, "test_loss": 0.11568929290771485}, {"step": 3300, "epoch": 7, "train_loss": 0.006756230723112822, "cos_r_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_innovation_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_apical_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_Ac_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "r_norm": [0.1959608942270279, 0.19186583161354065, 0.1897565722465515]}, {"step": 3400, "epoch": 7, "train_loss": 0.005082777701318264, "cos_r_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_innovation_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_apical_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_Ac_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "r_norm": [0.2011243999004364, 0.19720213115215302, 0.19440439343452454]}, {"step": 3500, "epoch": 7, "train_loss": 0.008085569366812706, "cos_r_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_innovation_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_apical_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_Ac_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "r_norm": [0.1824260950088501, 0.17886941134929657, 0.17662227153778076], "ldr": {"dL_sdil": -0.008172653615474701, "dL_bp": -0.011752061545848846, "ratio": 0.6954229760957564}}, {"step": 3600, "epoch": 7, "train_loss": 0.009758161380887032, "cos_r_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_innovation_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_apical_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_Ac_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "r_norm": [0.15041005611419678, 0.1479288935661316, 0.1454424113035202]}, {"step": 3700, "epoch": 7, "train_loss": 0.005388827063143253, "cos_r_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_innovation_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_apical_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_Ac_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "r_norm": [0.1920110434293747, 0.18857887387275696, 0.185125932097435]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9706, "test_loss": 0.12330467185974121}, {"step": 3800, "epoch": 8, "train_loss": 0.056129325181245804, "cos_r_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_innovation_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_apical_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_Ac_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "r_norm": [0.1895904839038849, 0.18607723712921143, 0.18282277882099152]}, {"step": 3900, "epoch": 8, "train_loss": 0.011785263195633888, "cos_r_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_innovation_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_apical_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_Ac_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "r_norm": [0.17273201048374176, 0.16941824555397034, 0.1666167974472046]}, {"step": 4000, "epoch": 8, "train_loss": 0.000689225154928863, "cos_r_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_innovation_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_apical_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_Ac_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "r_norm": [0.17630384862422943, 0.17299006879329681, 0.17047850787639618], "ldr": {"dL_sdil": -0.009058646857738495, "dL_bp": -0.014096729457378387, "ratio": 0.6426062786497684}}, {"step": 4100, "epoch": 8, "train_loss": 0.03716294467449188, "cos_r_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_innovation_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_apical_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_Ac_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "r_norm": [0.1756715625524521, 0.17255355417728424, 0.16945818066596985]}, {"step": 4200, "epoch": 8, "train_loss": 0.007642740849405527, "cos_r_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_innovation_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_apical_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_Ac_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "r_norm": [0.17260634899139404, 0.16935701668262482, 0.1664430946111679]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9715, "test_loss": 0.12795383539199828}, {"step": 4300, "epoch": 9, "train_loss": 0.005126616917550564, "cos_r_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_innovation_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_apical_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_Ac_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "r_norm": [0.19533851742744446, 0.19135543704032898, 0.1880948841571808]}, {"step": 4400, "epoch": 9, "train_loss": 0.003296743845567107, "cos_r_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_innovation_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_apical_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_Ac_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "r_norm": [0.18917936086654663, 0.1854909360408783, 0.18246984481811523]}, {"step": 4500, "epoch": 9, "train_loss": 0.0014693234115839005, "cos_r_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_innovation_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_apical_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_Ac_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "r_norm": [0.17914845049381256, 0.17544743418693542, 0.17231427133083344], "ldr": {"dL_sdil": -0.00979585200548172, "dL_bp": -0.01655876636505127, "ratio": 0.5915810266009143}}, {"step": 4600, "epoch": 9, "train_loss": 0.032596148550510406, "cos_r_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_innovation_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_apical_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_Ac_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "r_norm": [0.19013909995555878, 0.1861753761768341, 0.1826411783695221]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.974, "test_loss": 0.12575695457458497}, {"step": 4700, "epoch": 10, "train_loss": 0.0034915334545075893, "cos_r_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_innovation_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_apical_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_Ac_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "r_norm": [0.18801575899124146, 0.1843785047531128, 0.18106210231781006]}, {"step": 4800, "epoch": 10, "train_loss": 0.0008336055907420814, "cos_r_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_innovation_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_apical_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_Ac_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "r_norm": [0.1824166178703308, 0.17903029918670654, 0.17574921250343323]}, {"step": 4900, "epoch": 10, "train_loss": 0.001636673929169774, "cos_r_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_innovation_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_apical_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_Ac_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "r_norm": [0.17372174561023712, 0.17035624384880066, 0.16707497835159302]}, {"step": 5000, "epoch": 10, "train_loss": 0.0015556986909359694, "cos_r_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_innovation_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_apical_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_Ac_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "r_norm": [0.17774055898189545, 0.1746811866760254, 0.1715075671672821], "ldr": {"dL_sdil": -0.010479673743247986, "dL_bp": -0.01817304641008377, "ratio": 0.5766602641499378}}, {"step": 5100, "epoch": 10, "train_loss": 0.0015227128751575947, "cos_r_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_innovation_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_apical_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_Ac_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "r_norm": [0.18908870220184326, 0.18590903282165527, 0.18229123950004578]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9733, "test_loss": 0.12908906383514404}, {"step": 5200, "epoch": 11, "train_loss": 0.001309263869188726, "cos_r_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_innovation_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_apical_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_Ac_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "r_norm": [0.17493118345737457, 0.17182868719100952, 0.16859647631645203]}, {"step": 5300, "epoch": 11, "train_loss": 0.0011599198915064335, "cos_r_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_innovation_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_apical_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_Ac_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "r_norm": [0.16871681809425354, 0.16564862430095673, 0.16264042258262634]}, {"step": 5400, "epoch": 11, "train_loss": 0.0014726945664733648, "cos_r_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_innovation_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_apical_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_Ac_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "r_norm": [0.1738046109676361, 0.17061923444271088, 0.167450949549675]}, {"step": 5500, "epoch": 11, "train_loss": 0.0022923252545297146, "cos_r_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_innovation_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_apical_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_Ac_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "r_norm": [0.1730632185935974, 0.1701216697692871, 0.16676318645477295], "ldr": {"dL_sdil": -0.01092517375946045, "dL_bp": -0.019617542624473572, "ratio": 0.5569083737241846}}, {"step": 5600, "epoch": 11, "train_loss": 0.003013922367244959, "cos_r_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_innovation_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_apical_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_Ac_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "r_norm": [0.17643772065639496, 0.17336799204349518, 0.16997528076171875]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.974, "test_loss": 0.1334169048309326}, {"step": 5700, "epoch": 12, "train_loss": 0.00026223674649372697, "cos_r_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_innovation_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_apical_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_Ac_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "r_norm": [0.1758565604686737, 0.17264872789382935, 0.16945300996303558]}, {"step": 5800, "epoch": 12, "train_loss": 0.0003017765120603144, "cos_r_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_innovation_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_apical_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_Ac_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "r_norm": [0.17203187942504883, 0.1690245270729065, 0.16582567989826202]}, {"step": 5900, "epoch": 12, "train_loss": 0.0013039933983236551, "cos_r_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_innovation_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_apical_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_Ac_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "r_norm": [0.17366287112236023, 0.17064981162548065, 0.16749976575374603]}, {"step": 6000, "epoch": 12, "train_loss": 0.0008040090324357152, "cos_r_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_innovation_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_apical_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_Ac_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "r_norm": [0.17631205916404724, 0.17321062088012695, 0.17003434896469116], "ldr": {"dL_sdil": -0.011585041880607605, "dL_bp": -0.022001802921295166, "ratio": 0.526549661500451}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9745, "test_loss": 0.1352493793487549}, {"step": 6100, "epoch": 13, "train_loss": 0.0014463819097727537, "cos_r_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_innovation_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_apical_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_Ac_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "r_norm": [0.17908143997192383, 0.17593857645988464, 0.17247158288955688]}, {"step": 6200, "epoch": 13, "train_loss": 0.0002795365871861577, "cos_r_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_innovation_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_apical_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_Ac_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "r_norm": [0.17346340417861938, 0.17047497630119324, 0.16715440154075623]}, {"step": 6300, "epoch": 13, "train_loss": 0.0003245513653382659, "cos_r_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_innovation_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_apical_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_Ac_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "r_norm": [0.1769692301750183, 0.17405544221401215, 0.17055493593215942]}, {"step": 6400, "epoch": 13, "train_loss": 0.0006700620288029313, "cos_r_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_innovation_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_apical_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_Ac_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "r_norm": [0.17150437831878662, 0.16849254071712494, 0.16540826857089996]}, {"step": 6500, "epoch": 13, "train_loss": 0.00013033303548581898, "cos_r_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_innovation_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_apical_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_Ac_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "r_norm": [0.16887250542640686, 0.1660538762807846, 0.16295760869979858], "ldr": {"dL_sdil": -0.011667512357234955, "dL_bp": -0.022730067372322083, "ratio": 0.5133074251879357}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9737, "test_loss": 0.13770540838241577}, {"step": 6600, "epoch": 14, "train_loss": 0.002334980759769678, "cos_r_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_innovation_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_apical_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_Ac_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "r_norm": [0.1712503284215927, 0.16834032535552979, 0.16512611508369446]}, {"step": 6700, "epoch": 14, "train_loss": 0.0007730001234449446, "cos_r_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_innovation_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_apical_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_Ac_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "r_norm": [0.17286744713783264, 0.16990327835083008, 0.1666618287563324]}, {"step": 6800, "epoch": 14, "train_loss": 0.0018118484877049923, "cos_r_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_innovation_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_apical_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_Ac_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "r_norm": [0.17235802114009857, 0.16948559880256653, 0.166168212890625]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005928482860326767, "cos_r_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_innovation_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_apical_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_Ac_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "r_norm": [0.17221295833587646, 0.16921906173229218, 0.16597718000411987]}, {"step": 7000, "epoch": 14, "train_loss": 0.000632595329079777, "cos_r_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_innovation_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_apical_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_Ac_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "r_norm": [0.1718350499868393, 0.168907031416893, 0.16570332646369934], "ldr": {"dL_sdil": -0.012075692415237427, "dL_bp": -0.023587502539157867, "ratio": 0.5119529884603272}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9745, "test_loss": 0.14063884963989257}], "final": {"test_acc": 0.9745, "test_loss": 0.14063884963989257, "wall_s": 23.446836233139038, "cos_r_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_innovation_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_apical_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_Ac_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_matched_s1.json b/results/nuis_ctrl_v1_rho0_matched_s1.json new file mode 100644 index 0000000..ec80b7b --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_matched_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_matched_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.247864246368408, "cos_r_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_innovation_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_apical_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_Ac_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "r_norm": [4.729090213775635, 4.788191795349121, 4.781856536865234], "ldr": {"dL_sdil": -0.035402774810791016, "dL_bp": -0.2081589698791504, "ratio": 0.17007566299614468}}, {"step": 100, "epoch": 0, "train_loss": 0.3791123926639557, "cos_r_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_innovation_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_apical_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_Ac_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "r_norm": [0.8098099231719971, 0.8149547576904297, 0.8344215154647827]}, {"step": 200, "epoch": 0, "train_loss": 0.2855910360813141, "cos_r_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_innovation_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_apical_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_Ac_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "r_norm": [0.7800662517547607, 0.7833849787712097, 0.7984472513198853]}, {"step": 300, "epoch": 0, "train_loss": 0.3106156587600708, "cos_r_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_innovation_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_apical_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_Ac_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "r_norm": [0.6266008019447327, 0.6296441555023193, 0.6423605680465698]}, {"step": 400, "epoch": 0, "train_loss": 0.1368972212076187, "cos_r_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_innovation_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_apical_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_Ac_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "r_norm": [0.4948488473892212, 0.49738022685050964, 0.5028847455978394]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9471, "test_loss": 0.18460173034667968}, {"step": 500, "epoch": 1, "train_loss": 0.22857800126075745, "cos_r_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_innovation_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_apical_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_Ac_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "r_norm": [0.5006556510925293, 0.501956582069397, 0.5072718858718872], "ldr": {"dL_sdil": -0.010978296399116516, "dL_bp": -0.00958406925201416, "ratio": 1.1454734007487841}}, {"step": 600, "epoch": 1, "train_loss": 0.26168546080589294, "cos_r_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_innovation_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_apical_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_Ac_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "r_norm": [0.4931199550628662, 0.49525341391563416, 0.5031353235244751]}, {"step": 700, "epoch": 1, "train_loss": 0.08662884682416916, "cos_r_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_innovation_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_apical_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_Ac_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "r_norm": [0.41211724281311035, 0.4123530983924866, 0.42171144485473633]}, {"step": 800, "epoch": 1, "train_loss": 0.06552650779485703, "cos_r_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_innovation_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_apical_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_Ac_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "r_norm": [0.37752482295036316, 0.37500566244125366, 0.38382813334465027]}, {"step": 900, "epoch": 1, "train_loss": 0.16771306097507477, "cos_r_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_innovation_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_apical_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_Ac_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "r_norm": [0.3334032893180847, 0.33176112174987793, 0.3353516459465027]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9596, "test_loss": 0.1381149112701416}, {"step": 1000, "epoch": 2, "train_loss": 0.0976904109120369, "cos_r_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_innovation_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_apical_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_Ac_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "r_norm": [0.3129379451274872, 0.31030431389808655, 0.3134283423423767], "ldr": {"dL_sdil": -0.005524396896362305, "dL_bp": -0.00501541793346405, "ratio": 1.101482861378755}}, {"step": 1100, "epoch": 2, "train_loss": 0.11463465541601181, "cos_r_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_innovation_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_apical_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_Ac_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "r_norm": [0.27226677536964417, 0.27177441120147705, 0.2737849950790405]}, {"step": 1200, "epoch": 2, "train_loss": 0.022953230887651443, "cos_r_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_innovation_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_apical_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_Ac_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "r_norm": [0.3118118643760681, 0.31061288714408875, 0.3118583559989929]}, {"step": 1300, "epoch": 2, "train_loss": 0.19519557058811188, "cos_r_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_innovation_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_apical_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_Ac_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "r_norm": [0.27181971073150635, 0.26876047253608704, 0.2751275300979614]}, {"step": 1400, "epoch": 2, "train_loss": 0.20355364680290222, "cos_r_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_innovation_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_apical_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_Ac_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "r_norm": [0.26793360710144043, 0.2665303349494934, 0.2706131637096405]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9619, "test_loss": 0.12944196891784668}, {"step": 1500, "epoch": 3, "train_loss": 0.04727518558502197, "cos_r_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_innovation_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_apical_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_Ac_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "r_norm": [0.2907962203025818, 0.2895806133747101, 0.2904374301433563], "ldr": {"dL_sdil": -0.00766473263502121, "dL_bp": -0.007067129015922546, "ratio": 1.0845610173172495}}, {"step": 1600, "epoch": 3, "train_loss": 0.028240162879228592, "cos_r_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_innovation_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_apical_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_Ac_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "r_norm": [0.24258095026016235, 0.2405192106962204, 0.2429429292678833]}, {"step": 1700, "epoch": 3, "train_loss": 0.08582475781440735, "cos_r_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_innovation_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_apical_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_Ac_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "r_norm": [0.23459598422050476, 0.23319926857948303, 0.23500296473503113]}, {"step": 1800, "epoch": 3, "train_loss": 0.054279766976833344, "cos_r_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_innovation_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_apical_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_Ac_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "r_norm": [0.2751310467720032, 0.2752900719642639, 0.2777182459831238]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9686, "test_loss": 0.1130959093093872}, {"step": 1900, "epoch": 4, "train_loss": 0.08464933186769485, "cos_r_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_innovation_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_apical_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_Ac_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "r_norm": [0.21713416278362274, 0.21652851998806, 0.21772976219654083]}, {"step": 2000, "epoch": 4, "train_loss": 0.08816886693239212, "cos_r_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_innovation_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_apical_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_Ac_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "r_norm": [0.20291899144649506, 0.2031886875629425, 0.20419245958328247], "ldr": {"dL_sdil": -0.004766151309013367, "dL_bp": -0.005502060055732727, "ratio": 0.8662485070564434}}, {"step": 2100, "epoch": 4, "train_loss": 0.07242057472467422, "cos_r_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_innovation_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_apical_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_Ac_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "r_norm": [0.23937690258026123, 0.23910373449325562, 0.2406611144542694]}, {"step": 2200, "epoch": 4, "train_loss": 0.07056484371423721, "cos_r_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_innovation_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_apical_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_Ac_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "r_norm": [0.23531854152679443, 0.23204778134822845, 0.23232631385326385]}, {"step": 2300, "epoch": 4, "train_loss": 0.12437652051448822, "cos_r_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_innovation_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_apical_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_Ac_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "r_norm": [0.21436047554016113, 0.21310736238956451, 0.21543720364570618]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.967, "test_loss": 0.11917487754821778}, {"step": 2400, "epoch": 5, "train_loss": 0.02945767343044281, "cos_r_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_innovation_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_apical_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_Ac_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "r_norm": [0.17013323307037354, 0.16899538040161133, 0.17079725861549377]}, {"step": 2500, "epoch": 5, "train_loss": 0.05113217979669571, "cos_r_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_innovation_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_apical_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_Ac_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "r_norm": [0.19201859831809998, 0.190620556473732, 0.19072513282299042], "ldr": {"dL_sdil": -0.0062797218561172485, "dL_bp": -0.007466904819011688, "ratio": 0.8410073528898184}}, {"step": 2600, "epoch": 5, "train_loss": 0.00991781149059534, "cos_r_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_innovation_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_apical_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_Ac_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "r_norm": [0.19189369678497314, 0.1880035698413849, 0.18973243236541748]}, {"step": 2700, "epoch": 5, "train_loss": 0.05675402656197548, "cos_r_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_innovation_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_apical_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_Ac_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "r_norm": [0.2130669355392456, 0.20993337035179138, 0.21378093957901]}, {"step": 2800, "epoch": 5, "train_loss": 0.009208951145410538, "cos_r_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_innovation_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_apical_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_Ac_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "r_norm": [0.21457375586032867, 0.21203386783599854, 0.21387223899364471]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9724, "test_loss": 0.10937111053466797}, {"step": 2900, "epoch": 6, "train_loss": 0.014543101191520691, "cos_r_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_innovation_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_apical_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_Ac_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "r_norm": [0.18077343702316284, 0.1798022985458374, 0.18212389945983887]}, {"step": 3000, "epoch": 6, "train_loss": 0.006073781289160252, "cos_r_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_innovation_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_apical_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_Ac_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "r_norm": [0.15268780291080475, 0.15006791055202484, 0.1529131680727005], "ldr": {"dL_sdil": -0.005284003913402557, "dL_bp": -0.006912320852279663, "ratio": 0.7644326741082784}}, {"step": 3100, "epoch": 6, "train_loss": 0.07234150171279907, "cos_r_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_innovation_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_apical_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_Ac_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "r_norm": [0.13482704758644104, 0.13283531367778778, 0.1336984634399414]}, {"step": 3200, "epoch": 6, "train_loss": 0.06304893642663956, "cos_r_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_innovation_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_apical_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_Ac_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "r_norm": [0.18191789090633392, 0.1812913715839386, 0.18045282363891602]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9716, "test_loss": 0.11546330757141113}, {"step": 3300, "epoch": 7, "train_loss": 0.008135353215038776, "cos_r_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_innovation_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_apical_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_Ac_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "r_norm": [0.1324104368686676, 0.13133256137371063, 0.1319148689508438]}, {"step": 3400, "epoch": 7, "train_loss": 0.003062707372009754, "cos_r_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_innovation_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_apical_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_Ac_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "r_norm": [0.17751097679138184, 0.17610973119735718, 0.17820629477500916]}, {"step": 3500, "epoch": 7, "train_loss": 0.0075003234669566154, "cos_r_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_innovation_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_apical_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_Ac_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "r_norm": [0.15693698823451996, 0.1564456969499588, 0.15661953389644623], "ldr": {"dL_sdil": -0.005416072905063629, "dL_bp": -0.0077264681458473206, "ratio": 0.7009765397110399}}, {"step": 3600, "epoch": 7, "train_loss": 0.004378090146929026, "cos_r_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_innovation_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_apical_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_Ac_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "r_norm": [0.15218180418014526, 0.15111035108566284, 0.1530647873878479]}, {"step": 3700, "epoch": 7, "train_loss": 0.005401092581450939, "cos_r_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_innovation_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_apical_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_Ac_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "r_norm": [0.15374818444252014, 0.1511906236410141, 0.15268918871879578]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9725, "test_loss": 0.12304098224639892}, {"step": 3800, "epoch": 8, "train_loss": 0.020612072199583054, "cos_r_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_innovation_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_apical_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_Ac_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "r_norm": [0.18328630924224854, 0.18096177279949188, 0.1839454621076584]}, {"step": 3900, "epoch": 8, "train_loss": 0.00907292403280735, "cos_r_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_innovation_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_apical_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_Ac_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "r_norm": [0.15745952725410461, 0.15553179383277893, 0.15759898722171783]}, {"step": 4000, "epoch": 8, "train_loss": 0.0011170797515660524, "cos_r_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_innovation_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_apical_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_Ac_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "r_norm": [0.13805150985717773, 0.13616320490837097, 0.1368236541748047], "ldr": {"dL_sdil": -0.005435429513454437, "dL_bp": -0.00862833857536316, "ratio": 0.6299508840525146}}, {"step": 4100, "epoch": 8, "train_loss": 0.06657039374113083, "cos_r_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_innovation_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_apical_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_Ac_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "r_norm": [0.14812925457954407, 0.14660516381263733, 0.14825358986854553]}, {"step": 4200, "epoch": 8, "train_loss": 0.01757517084479332, "cos_r_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_innovation_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_apical_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_Ac_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "r_norm": [0.14519406855106354, 0.14317519962787628, 0.1447223722934723]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9733, "test_loss": 0.12583541717529298}, {"step": 4300, "epoch": 9, "train_loss": 0.0043710931204259396, "cos_r_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_innovation_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_apical_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_Ac_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "r_norm": [0.17884290218353271, 0.1771182417869568, 0.17936065793037415]}, {"step": 4400, "epoch": 9, "train_loss": 0.012221035547554493, "cos_r_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_innovation_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_apical_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_Ac_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "r_norm": [0.159407377243042, 0.15782558917999268, 0.15985700488090515]}, {"step": 4500, "epoch": 9, "train_loss": 0.004442101344466209, "cos_r_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_innovation_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_apical_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_Ac_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "r_norm": [0.1378084421157837, 0.13638418912887573, 0.13881981372833252], "ldr": {"dL_sdil": -0.006090611219406128, "dL_bp": -0.009796351194381714, "ratio": 0.6217224248656115}}, {"step": 4600, "epoch": 9, "train_loss": 0.05368257686495781, "cos_r_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_innovation_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_apical_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_Ac_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "r_norm": [0.14086458086967468, 0.1384350061416626, 0.1409018635749817]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9722, "test_loss": 0.1313097234725952}, {"step": 4700, "epoch": 10, "train_loss": 0.0048868609592318535, "cos_r_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_innovation_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_apical_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_Ac_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "r_norm": [0.13858553767204285, 0.13714627921581268, 0.13974396884441376]}, {"step": 4800, "epoch": 10, "train_loss": 0.001804178231395781, "cos_r_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_innovation_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_apical_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_Ac_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "r_norm": [0.1522747427225113, 0.1504199206829071, 0.15280333161354065]}, {"step": 4900, "epoch": 10, "train_loss": 0.0028787143528461456, "cos_r_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_innovation_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_apical_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_Ac_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "r_norm": [0.1442190408706665, 0.14268440008163452, 0.14490017294883728]}, {"step": 5000, "epoch": 10, "train_loss": 0.0013339570723474026, "cos_r_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_innovation_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_apical_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_Ac_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "r_norm": [0.1472242772579193, 0.14609915018081665, 0.14829722046852112], "ldr": {"dL_sdil": -0.006713069975376129, "dL_bp": -0.011234067380428314, "ratio": 0.5975636203741715}}, {"step": 5100, "epoch": 10, "train_loss": 0.0018756217323243618, "cos_r_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_innovation_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_apical_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_Ac_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "r_norm": [0.14868545532226562, 0.14725178480148315, 0.14963632822036743]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.973, "test_loss": 0.13351503677368165}, {"step": 5200, "epoch": 11, "train_loss": 0.0014711369294673204, "cos_r_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_innovation_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_apical_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_Ac_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "r_norm": [0.12346100807189941, 0.12253932654857635, 0.12461203336715698]}, {"step": 5300, "epoch": 11, "train_loss": 0.001629142090678215, "cos_r_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_innovation_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_apical_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_Ac_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "r_norm": [0.12968139350414276, 0.1281958818435669, 0.13058608770370483]}, {"step": 5400, "epoch": 11, "train_loss": 0.0024097473360598087, "cos_r_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_innovation_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_apical_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_Ac_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "r_norm": [0.12974487245082855, 0.12847599387168884, 0.13078004121780396]}, {"step": 5500, "epoch": 11, "train_loss": 0.0010489317355677485, "cos_r_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_innovation_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_apical_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_Ac_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "r_norm": [0.13474895060062408, 0.13274598121643066, 0.13516360521316528], "ldr": {"dL_sdil": -0.006466925144195557, "dL_bp": -0.01084946095943451, "ratio": 0.5960595801372074}}, {"step": 5600, "epoch": 11, "train_loss": 0.0009077594731934369, "cos_r_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_innovation_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_apical_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_Ac_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "r_norm": [0.13371193408966064, 0.13146626949310303, 0.13360051810741425]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9739, "test_loss": 0.1354274552345276}, {"step": 5700, "epoch": 12, "train_loss": 0.0005193198448978364, "cos_r_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_innovation_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_apical_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_Ac_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "r_norm": [0.13641861081123352, 0.13459017872810364, 0.1368558406829834]}, {"step": 5800, "epoch": 12, "train_loss": 0.0007246336317621171, "cos_r_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_innovation_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_apical_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_Ac_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "r_norm": [0.1397680640220642, 0.137950599193573, 0.14015179872512817]}, {"step": 5900, "epoch": 12, "train_loss": 0.0012115520657971501, "cos_r_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_innovation_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_apical_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_Ac_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "r_norm": [0.12886938452720642, 0.12738792598247528, 0.1296568214893341]}, {"step": 6000, "epoch": 12, "train_loss": 0.0011408575810492039, "cos_r_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_innovation_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_apical_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_Ac_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "r_norm": [0.13654574751853943, 0.13504381477832794, 0.1375402808189392], "ldr": {"dL_sdil": -0.006541714072227478, "dL_bp": -0.010865777730941772, "ratio": 0.602047477337868}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9739, "test_loss": 0.13652948637008666}, {"step": 6100, "epoch": 13, "train_loss": 0.0016530295833945274, "cos_r_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_innovation_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_apical_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_Ac_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "r_norm": [0.13538393378257751, 0.13388687372207642, 0.1362152248620987]}, {"step": 6200, "epoch": 13, "train_loss": 0.0004883500514551997, "cos_r_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_innovation_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_apical_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_Ac_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "r_norm": [0.13575664162635803, 0.13420100510120392, 0.13645632565021515]}, {"step": 6300, "epoch": 13, "train_loss": 0.0005272991256788373, "cos_r_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_innovation_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_apical_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_Ac_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "r_norm": [0.1379033327102661, 0.1362687349319458, 0.13877828419208527]}, {"step": 6400, "epoch": 13, "train_loss": 0.0008385080145671964, "cos_r_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_innovation_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_apical_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_Ac_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "r_norm": [0.1363997906446457, 0.1347801685333252, 0.13696125149726868]}, {"step": 6500, "epoch": 13, "train_loss": 0.0004538414941634983, "cos_r_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_innovation_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_apical_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_Ac_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "r_norm": [0.13410302996635437, 0.13246551156044006, 0.1348181515932083], "ldr": {"dL_sdil": -0.006781101226806641, "dL_bp": -0.01180441677570343, "ratio": 0.5744545754063781}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9737, "test_loss": 0.13901660509109498}, {"step": 6600, "epoch": 14, "train_loss": 0.0003573036228772253, "cos_r_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_innovation_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_apical_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_Ac_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "r_norm": [0.13315507769584656, 0.13159938156604767, 0.1339610517024994]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008824412361718714, "cos_r_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_innovation_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_apical_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_Ac_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "r_norm": [0.1327393501996994, 0.1310739368200302, 0.13362014293670654]}, {"step": 6800, "epoch": 14, "train_loss": 0.0010087769478559494, "cos_r_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_innovation_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_apical_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_Ac_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "r_norm": [0.13503387570381165, 0.1333080381155014, 0.13559962809085846]}, {"step": 6900, "epoch": 14, "train_loss": 0.0009153951541520655, "cos_r_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_innovation_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_apical_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_Ac_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "r_norm": [0.13242390751838684, 0.1308332085609436, 0.13339316844940186]}, {"step": 7000, "epoch": 14, "train_loss": 0.00029737711884081364, "cos_r_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_innovation_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_apical_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_Ac_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "r_norm": [0.1377774327993393, 0.1363024264574051, 0.13875912129878998], "ldr": {"dL_sdil": -0.007288441061973572, "dL_bp": -0.01291123777627945, "ratio": 0.564503666361401}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9742, "test_loss": 0.14128447875976563}], "final": {"test_acc": 0.9742, "test_loss": 0.14128447875976563, "wall_s": 23.92648673057556, "cos_r_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_innovation_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_apical_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_Ac_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_matched_s2.json b/results/nuis_ctrl_v1_rho0_matched_s2.json new file mode 100644 index 0000000..4204df9 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_matched_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_matched_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4856414794921875, "cos_r_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_innovation_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_apical_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_Ac_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "r_norm": [4.7544169425964355, 4.779862403869629, 4.861393451690674], "ldr": {"dL_sdil": -0.044302940368652344, "dL_bp": -0.24401354789733887, "ratio": 0.18155934680844618}}, {"step": 100, "epoch": 0, "train_loss": 0.3662866950035095, "cos_r_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_innovation_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_apical_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_Ac_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "r_norm": [0.7926816940307617, 0.8032606840133667, 0.8174332976341248]}, {"step": 200, "epoch": 0, "train_loss": 0.2967141568660736, "cos_r_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_innovation_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_apical_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_Ac_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "r_norm": [0.724601149559021, 0.7336956262588501, 0.7452678680419922]}, {"step": 300, "epoch": 0, "train_loss": 0.2878264784812927, "cos_r_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_innovation_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_apical_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_Ac_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "r_norm": [0.6228486895561218, 0.6298609972000122, 0.6399674415588379]}, {"step": 400, "epoch": 0, "train_loss": 0.13541436195373535, "cos_r_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_innovation_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_apical_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_Ac_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "r_norm": [0.5261503458023071, 0.5319482684135437, 0.5407342910766602]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9436, "test_loss": 0.18730589828491212}, {"step": 500, "epoch": 1, "train_loss": 0.2773134708404541, "cos_r_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_innovation_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_apical_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_Ac_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "r_norm": [0.5363684892654419, 0.5415335893630981, 0.5485605597496033], "ldr": {"dL_sdil": -0.016582071781158447, "dL_bp": -0.013470649719238281, "ratio": 1.2309778761061947}}, {"step": 600, "epoch": 1, "train_loss": 0.24808740615844727, "cos_r_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_innovation_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_apical_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_Ac_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "r_norm": [0.4696118235588074, 0.4752689301967621, 0.48475584387779236]}, {"step": 700, "epoch": 1, "train_loss": 0.059591565281152725, "cos_r_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_innovation_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_apical_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_Ac_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "r_norm": [0.4132225215435028, 0.417829692363739, 0.42398762702941895]}, {"step": 800, "epoch": 1, "train_loss": 0.08955289423465729, "cos_r_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_innovation_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_apical_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_Ac_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "r_norm": [0.38012179732322693, 0.38466107845306396, 0.3904116451740265]}, {"step": 900, "epoch": 1, "train_loss": 0.12817199528217316, "cos_r_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_innovation_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_apical_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_Ac_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "r_norm": [0.42234253883361816, 0.42529693245887756, 0.43161889910697937]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9563, "test_loss": 0.14741420364379884}, {"step": 1000, "epoch": 2, "train_loss": 0.15121077001094818, "cos_r_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_innovation_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_apical_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_Ac_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "r_norm": [0.38415396213531494, 0.38890281319618225, 0.3945630192756653], "ldr": {"dL_sdil": -0.01034557819366455, "dL_bp": -0.009327605366706848, "ratio": 1.109135494796035}}, {"step": 1100, "epoch": 2, "train_loss": 0.05521024763584137, "cos_r_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_innovation_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_apical_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_Ac_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "r_norm": [0.2886888384819031, 0.2918413281440735, 0.29741814732551575]}, {"step": 1200, "epoch": 2, "train_loss": 0.06215231865644455, "cos_r_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_innovation_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_apical_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_Ac_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "r_norm": [0.32788410782814026, 0.33229193091392517, 0.3380418121814728]}, {"step": 1300, "epoch": 2, "train_loss": 0.18930861353874207, "cos_r_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_innovation_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_apical_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_Ac_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "r_norm": [0.33047181367874146, 0.334816038608551, 0.3400202989578247]}, {"step": 1400, "epoch": 2, "train_loss": 0.20265927910804749, "cos_r_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_innovation_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_apical_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_Ac_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "r_norm": [0.3220111131668091, 0.32452118396759033, 0.3309333324432373]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9623, "test_loss": 0.1362317768096924}, {"step": 1500, "epoch": 3, "train_loss": 0.054935283958911896, "cos_r_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_innovation_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_apical_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_Ac_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "r_norm": [0.29881566762924194, 0.30197426676750183, 0.3079264760017395], "ldr": {"dL_sdil": -0.008590251207351685, "dL_bp": -0.008395388722419739, "ratio": 1.0232106566324402}}, {"step": 1600, "epoch": 3, "train_loss": 0.044838763773441315, "cos_r_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_innovation_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_apical_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_Ac_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "r_norm": [0.26855069398880005, 0.2721232771873474, 0.27481386065483093]}, {"step": 1700, "epoch": 3, "train_loss": 0.08930075168609619, "cos_r_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_innovation_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_apical_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_Ac_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "r_norm": [0.2840762436389923, 0.2851904630661011, 0.29301607608795166]}, {"step": 1800, "epoch": 3, "train_loss": 0.08093765377998352, "cos_r_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_innovation_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_apical_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_Ac_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "r_norm": [0.3048495054244995, 0.3076798915863037, 0.31106480956077576]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.968, "test_loss": 0.11007094802856446}, {"step": 1900, "epoch": 4, "train_loss": 0.04483484849333763, "cos_r_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_innovation_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_apical_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_Ac_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "r_norm": [0.22422946989536285, 0.22653135657310486, 0.23203539848327637]}, {"step": 2000, "epoch": 4, "train_loss": 0.06511980295181274, "cos_r_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_innovation_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_apical_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_Ac_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "r_norm": [0.22974814474582672, 0.2313867211341858, 0.23618879914283752], "ldr": {"dL_sdil": -0.005495421588420868, "dL_bp": -0.0060300081968307495, "ratio": 0.9113456249212315}}, {"step": 2100, "epoch": 4, "train_loss": 0.033010225743055344, "cos_r_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_innovation_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_apical_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_Ac_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "r_norm": [0.25794345140457153, 0.260486364364624, 0.26540467143058777]}, {"step": 2200, "epoch": 4, "train_loss": 0.10495644807815552, "cos_r_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_innovation_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_apical_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_Ac_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "r_norm": [0.24134549498558044, 0.2427096962928772, 0.2470351755619049]}, {"step": 2300, "epoch": 4, "train_loss": 0.13655468821525574, "cos_r_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_innovation_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_apical_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_Ac_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "r_norm": [0.2497454583644867, 0.2505592703819275, 0.25532519817352295]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.971, "test_loss": 0.1055252311706543}, {"step": 2400, "epoch": 5, "train_loss": 0.03528325632214546, "cos_r_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_innovation_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_apical_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_Ac_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "r_norm": [0.2337566614151001, 0.23517361283302307, 0.24056139588356018]}, {"step": 2500, "epoch": 5, "train_loss": 0.051970984786748886, "cos_r_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_innovation_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_apical_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_Ac_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "r_norm": [0.20650900900363922, 0.20780494809150696, 0.21189066767692566], "ldr": {"dL_sdil": -0.0069409385323524475, "dL_bp": -0.00879254937171936, "ratio": 0.7894113798982473}}, {"step": 2600, "epoch": 5, "train_loss": 0.018195580691099167, "cos_r_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_innovation_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_apical_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_Ac_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "r_norm": [0.19724297523498535, 0.1984039545059204, 0.2021041214466095]}, {"step": 2700, "epoch": 5, "train_loss": 0.0936257466673851, "cos_r_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_innovation_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_apical_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_Ac_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "r_norm": [0.20749273896217346, 0.209029883146286, 0.21408987045288086]}, {"step": 2800, "epoch": 5, "train_loss": 0.02398904599249363, "cos_r_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_innovation_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_apical_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_Ac_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "r_norm": [0.2375624030828476, 0.2401655614376068, 0.2447095513343811]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9716, "test_loss": 0.1127711856842041}, {"step": 2900, "epoch": 6, "train_loss": 0.02472660131752491, "cos_r_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_innovation_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_apical_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_Ac_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "r_norm": [0.200751394033432, 0.20256194472312927, 0.20603583753108978]}, {"step": 3000, "epoch": 6, "train_loss": 0.01555752381682396, "cos_r_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_innovation_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_apical_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_Ac_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "r_norm": [0.22714081406593323, 0.22868841886520386, 0.2330799400806427], "ldr": {"dL_sdil": -0.008373714983463287, "dL_bp": -0.011146388947963715, "ratio": 0.7512491285344071}}, {"step": 3100, "epoch": 6, "train_loss": 0.012355584651231766, "cos_r_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_innovation_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_apical_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_Ac_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "r_norm": [0.19545826315879822, 0.1969728022813797, 0.19908244907855988]}, {"step": 3200, "epoch": 6, "train_loss": 0.03212413936853409, "cos_r_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_innovation_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_apical_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_Ac_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "r_norm": [0.20170357823371887, 0.20397330820560455, 0.20707929134368896]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9719, "test_loss": 0.11250891075134277}, {"step": 3300, "epoch": 7, "train_loss": 0.014137478545308113, "cos_r_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_innovation_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_apical_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_Ac_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "r_norm": [0.19205130636692047, 0.19374202191829681, 0.1985575556755066]}, {"step": 3400, "epoch": 7, "train_loss": 0.015627946704626083, "cos_r_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_innovation_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_apical_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_Ac_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "r_norm": [0.17368774116039276, 0.17372958362102509, 0.17921704053878784]}, {"step": 3500, "epoch": 7, "train_loss": 0.009410088881850243, "cos_r_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_innovation_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_apical_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_Ac_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "r_norm": [0.15606719255447388, 0.1569913625717163, 0.16070200502872467], "ldr": {"dL_sdil": -0.005623310804367065, "dL_bp": -0.008359216153621674, "ratio": 0.6727079071798779}}, {"step": 3600, "epoch": 7, "train_loss": 0.007804212160408497, "cos_r_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_innovation_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_apical_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_Ac_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "r_norm": [0.1693708598613739, 0.16993819177150726, 0.17434334754943848]}, {"step": 3700, "epoch": 7, "train_loss": 0.002727900166064501, "cos_r_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_innovation_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_apical_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_Ac_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "r_norm": [0.19530899822711945, 0.1965683102607727, 0.20004117488861084]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9735, "test_loss": 0.12146724281311035}, {"step": 3800, "epoch": 8, "train_loss": 0.0279134102165699, "cos_r_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_innovation_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_apical_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_Ac_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "r_norm": [0.199128657579422, 0.1991393268108368, 0.20451344549655914]}, {"step": 3900, "epoch": 8, "train_loss": 0.007016683928668499, "cos_r_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_innovation_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_apical_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_Ac_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "r_norm": [0.17034371197223663, 0.17111225426197052, 0.17593686282634735]}, {"step": 4000, "epoch": 8, "train_loss": 0.0011241799220442772, "cos_r_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_innovation_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_apical_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_Ac_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "r_norm": [0.2055906504392624, 0.206048384308815, 0.2106199562549591], "ldr": {"dL_sdil": -0.009515531361103058, "dL_bp": -0.01432783156633377, "ratio": 0.6641292031560299}}, {"step": 4100, "epoch": 8, "train_loss": 0.052838996052742004, "cos_r_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_innovation_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_apical_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_Ac_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "r_norm": [0.18898266553878784, 0.1900315284729004, 0.19334247708320618]}, {"step": 4200, "epoch": 8, "train_loss": 0.010721303522586823, "cos_r_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_innovation_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_apical_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_Ac_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "r_norm": [0.1758223921060562, 0.17690947651863098, 0.1805993616580963]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9726, "test_loss": 0.12696008186340332}, {"step": 4300, "epoch": 9, "train_loss": 0.0010635580401867628, "cos_r_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_innovation_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_apical_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_Ac_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "r_norm": [0.17538368701934814, 0.17650076746940613, 0.1804800033569336]}, {"step": 4400, "epoch": 9, "train_loss": 0.0015249300049617887, "cos_r_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_innovation_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_apical_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_Ac_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "r_norm": [0.17721575498580933, 0.17846234142780304, 0.1818528175354004]}, {"step": 4500, "epoch": 9, "train_loss": 0.005601717159152031, "cos_r_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_innovation_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_apical_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_Ac_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "r_norm": [0.16339799761772156, 0.16483958065509796, 0.16933569312095642], "ldr": {"dL_sdil": -0.0075685083866119385, "dL_bp": -0.012692451477050781, "ratio": 0.5962999661882936}}, {"step": 4600, "epoch": 9, "train_loss": 0.04552677646279335, "cos_r_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_innovation_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_apical_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_Ac_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "r_norm": [0.16808894276618958, 0.16934075951576233, 0.1726376712322235]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9747, "test_loss": 0.12921665706634522}, {"step": 4700, "epoch": 10, "train_loss": 0.0016382120084017515, "cos_r_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_innovation_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_apical_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_Ac_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "r_norm": [0.17644554376602173, 0.17762485146522522, 0.18118296563625336]}, {"step": 4800, "epoch": 10, "train_loss": 0.0031079151667654514, "cos_r_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_innovation_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_apical_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_Ac_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "r_norm": [0.17002223432064056, 0.17095139622688293, 0.1749536097049713]}, {"step": 4900, "epoch": 10, "train_loss": 0.001899699796922505, "cos_r_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_innovation_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_apical_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_Ac_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "r_norm": [0.1906973421573639, 0.19160214066505432, 0.19531694054603577]}, {"step": 5000, "epoch": 10, "train_loss": 0.002346828579902649, "cos_r_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_innovation_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_apical_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_Ac_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "r_norm": [0.17863205075263977, 0.18014976382255554, 0.18399974703788757], "ldr": {"dL_sdil": -0.008974842727184296, "dL_bp": -0.015554085373878479, "ratio": 0.5770087093810505}}, {"step": 5100, "epoch": 10, "train_loss": 0.0018325410783290863, "cos_r_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_innovation_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_apical_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_Ac_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "r_norm": [0.17877718806266785, 0.1797236204147339, 0.183450847864151]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9742, "test_loss": 0.1357520221710205}, {"step": 5200, "epoch": 11, "train_loss": 0.0008780563366599381, "cos_r_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_innovation_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_apical_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_Ac_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "r_norm": [0.17967163026332855, 0.18078157305717468, 0.1842673122882843]}, {"step": 5300, "epoch": 11, "train_loss": 0.0013086458202451468, "cos_r_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_innovation_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_apical_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_Ac_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "r_norm": [0.17678910493850708, 0.1777363121509552, 0.18114155530929565]}, {"step": 5400, "epoch": 11, "train_loss": 0.0006866963813081384, "cos_r_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_innovation_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_apical_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_Ac_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "r_norm": [0.18510422110557556, 0.18618690967559814, 0.1894882768392563]}, {"step": 5500, "epoch": 11, "train_loss": 0.0008870868477970362, "cos_r_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_innovation_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_apical_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_Ac_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "r_norm": [0.17689445614814758, 0.17830178141593933, 0.18123194575309753], "ldr": {"dL_sdil": -0.008988067507743835, "dL_bp": -0.01625603437423706, "ratio": 0.5529065269441318}}, {"step": 5600, "epoch": 11, "train_loss": 0.0004585091082844883, "cos_r_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_innovation_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_apical_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_Ac_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "r_norm": [0.17601367831230164, 0.17745786905288696, 0.18061557412147522]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9754, "test_loss": 0.13874979991912842}, {"step": 5700, "epoch": 12, "train_loss": 0.0007273177616298199, "cos_r_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_innovation_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_apical_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_Ac_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "r_norm": [0.1876221001148224, 0.18882854282855988, 0.1921488344669342]}, {"step": 5800, "epoch": 12, "train_loss": 0.00034042136394418776, "cos_r_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_innovation_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_apical_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_Ac_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "r_norm": [0.1746070384979248, 0.17565712332725525, 0.17919966578483582]}, {"step": 5900, "epoch": 12, "train_loss": 0.0014644403709098697, "cos_r_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_innovation_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_apical_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_Ac_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "r_norm": [0.171538308262825, 0.17262454330921173, 0.176008403301239]}, {"step": 6000, "epoch": 12, "train_loss": 0.0009379165712743998, "cos_r_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_innovation_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_apical_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_Ac_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "r_norm": [0.17355795204639435, 0.17495247721672058, 0.17821210622787476], "ldr": {"dL_sdil": -0.008706547319889069, "dL_bp": -0.015992946922779083, "ratio": 0.5443991880876035}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9754, "test_loss": 0.14179479808807374}, {"step": 6100, "epoch": 13, "train_loss": 0.001136131351813674, "cos_r_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_innovation_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_apical_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_Ac_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "r_norm": [0.17050513625144958, 0.1718040406703949, 0.17503532767295837]}, {"step": 6200, "epoch": 13, "train_loss": 0.00029565332806669176, "cos_r_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_innovation_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_apical_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_Ac_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "r_norm": [0.1782611608505249, 0.17942413687705994, 0.18300361931324005]}, {"step": 6300, "epoch": 13, "train_loss": 0.00017874882905744016, "cos_r_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_innovation_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_apical_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_Ac_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "r_norm": [0.1737596094608307, 0.17501607537269592, 0.1786150336265564]}, {"step": 6400, "epoch": 13, "train_loss": 0.0014268780359998345, "cos_r_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_innovation_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_apical_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_Ac_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "r_norm": [0.17299015820026398, 0.17426249384880066, 0.1777229905128479]}, {"step": 6500, "epoch": 13, "train_loss": 0.00022807065397500992, "cos_r_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_innovation_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_apical_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_Ac_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "r_norm": [0.1683826446533203, 0.16965822875499725, 0.17293044924736023], "ldr": {"dL_sdil": -0.008848324418067932, "dL_bp": -0.017275981605052948, "ratio": 0.5121749154606612}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9744, "test_loss": 0.14613630256652832}, {"step": 6600, "epoch": 14, "train_loss": 0.0008223638869822025, "cos_r_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_innovation_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_apical_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_Ac_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "r_norm": [0.17072054743766785, 0.17188414931297302, 0.17549914121627808]}, {"step": 6700, "epoch": 14, "train_loss": 0.0012336993822827935, "cos_r_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_innovation_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_apical_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_Ac_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "r_norm": [0.17281609773635864, 0.17397551238536835, 0.1775282919406891]}, {"step": 6800, "epoch": 14, "train_loss": 0.0017774601001292467, "cos_r_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_innovation_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_apical_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_Ac_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "r_norm": [0.1771830916404724, 0.17863672971725464, 0.18169184029102325]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006432391237467527, "cos_r_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_innovation_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_apical_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_Ac_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "r_norm": [0.1692524403333664, 0.17056486010551453, 0.17403511703014374]}, {"step": 7000, "epoch": 14, "train_loss": 0.000249928911216557, "cos_r_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_innovation_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_apical_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_Ac_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "r_norm": [0.1676335632801056, 0.16881629824638367, 0.17234541475772858], "ldr": {"dL_sdil": -0.008975468575954437, "dL_bp": -0.017749518156051636, "ratio": 0.505673928556437}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9742, "test_loss": 0.14835414848327635}], "final": {"test_acc": 0.9742, "test_loss": 0.14835414848327635, "wall_s": 23.57845139503479, "cos_r_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_innovation_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_apical_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_Ac_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_matched_s3.json b/results/nuis_ctrl_v1_rho0_matched_s3.json new file mode 100644 index 0000000..1e3d168 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_matched_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_matched_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3283889293670654, "cos_r_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_innovation_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_apical_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_Ac_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "r_norm": [4.841709613800049, 4.737439155578613, 4.835896015167236], "ldr": {"dL_sdil": -0.0037169456481933594, "dL_bp": -0.22416257858276367, "ratio": 0.016581472570904674}}, {"step": 100, "epoch": 0, "train_loss": 0.33480948209762573, "cos_r_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_innovation_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_apical_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_Ac_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "r_norm": [0.7733307480812073, 0.758333683013916, 0.7694300413131714]}, {"step": 200, "epoch": 0, "train_loss": 0.32067805528640747, "cos_r_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_innovation_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_apical_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_Ac_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "r_norm": [0.7619251608848572, 0.7530097961425781, 0.7603987455368042]}, {"step": 300, "epoch": 0, "train_loss": 0.3700808882713318, "cos_r_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_innovation_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_apical_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_Ac_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "r_norm": [0.5746974945068359, 0.5715751647949219, 0.5759649276733398]}, {"step": 400, "epoch": 0, "train_loss": 0.11992092430591583, "cos_r_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_innovation_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_apical_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_Ac_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "r_norm": [0.4775196313858032, 0.47117429971694946, 0.4762042164802551]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9485, "test_loss": 0.17359396591186524}, {"step": 500, "epoch": 1, "train_loss": 0.25980252027511597, "cos_r_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_innovation_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_apical_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_Ac_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "r_norm": [0.5102050304412842, 0.5090956091880798, 0.5102525949478149], "ldr": {"dL_sdil": -0.014272913336753845, "dL_bp": -0.011847808957099915, "ratio": 1.2046880050509814}}, {"step": 600, "epoch": 1, "train_loss": 0.20059989392757416, "cos_r_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_innovation_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_apical_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_Ac_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "r_norm": [0.4354090094566345, 0.4269413948059082, 0.4325163960456848]}, {"step": 700, "epoch": 1, "train_loss": 0.08970798552036285, "cos_r_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_innovation_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_apical_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_Ac_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "r_norm": [0.4034784436225891, 0.4017947316169739, 0.4032314419746399]}, {"step": 800, "epoch": 1, "train_loss": 0.10161302983760834, "cos_r_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_innovation_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_apical_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_Ac_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "r_norm": [0.4015730917453766, 0.39593786001205444, 0.4005199074745178]}, {"step": 900, "epoch": 1, "train_loss": 0.12729158997535706, "cos_r_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_innovation_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_apical_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_Ac_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "r_norm": [0.3631213307380676, 0.3588687479496002, 0.36084190011024475]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9597, "test_loss": 0.1398345920562744}, {"step": 1000, "epoch": 2, "train_loss": 0.13055379688739777, "cos_r_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_innovation_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_apical_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_Ac_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "r_norm": [0.3458414077758789, 0.339888334274292, 0.3438340425491333], "ldr": {"dL_sdil": -0.00751737505197525, "dL_bp": -0.006968997418880463, "ratio": 1.0786881670538604}}, {"step": 1100, "epoch": 2, "train_loss": 0.08165145665407181, "cos_r_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_innovation_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_apical_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_Ac_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "r_norm": [0.27777719497680664, 0.27354109287261963, 0.2770301103591919]}, {"step": 1200, "epoch": 2, "train_loss": 0.03162624314427376, "cos_r_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_innovation_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_apical_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_Ac_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "r_norm": [0.3397935926914215, 0.3343070149421692, 0.33784517645835876]}, {"step": 1300, "epoch": 2, "train_loss": 0.12580201029777527, "cos_r_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_innovation_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_apical_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_Ac_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "r_norm": [0.27239900827407837, 0.26863694190979004, 0.2723345160484314]}, {"step": 1400, "epoch": 2, "train_loss": 0.20885255932807922, "cos_r_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_innovation_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_apical_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_Ac_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "r_norm": [0.28959929943084717, 0.28440648317337036, 0.2864704132080078]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9624, "test_loss": 0.127296443939209}, {"step": 1500, "epoch": 3, "train_loss": 0.07511237263679504, "cos_r_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_innovation_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_apical_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_Ac_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "r_norm": [0.31371599435806274, 0.30785977840423584, 0.31208980083465576], "ldr": {"dL_sdil": -0.007627889513969421, "dL_bp": -0.006990775465965271, "ratio": 1.0911363912495764}}, {"step": 1600, "epoch": 3, "train_loss": 0.05954476445913315, "cos_r_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_innovation_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_apical_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_Ac_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "r_norm": [0.2844116687774658, 0.28088948130607605, 0.2819110155105591]}, {"step": 1700, "epoch": 3, "train_loss": 0.05954836308956146, "cos_r_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_innovation_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_apical_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_Ac_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "r_norm": [0.27005547285079956, 0.2676847577095032, 0.2688477635383606]}, {"step": 1800, "epoch": 3, "train_loss": 0.04903425648808479, "cos_r_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_innovation_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_apical_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_Ac_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "r_norm": [0.2799452543258667, 0.2738724946975708, 0.27548953890800476]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9699, "test_loss": 0.10895987529754639}, {"step": 1900, "epoch": 4, "train_loss": 0.05801629275083542, "cos_r_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_innovation_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_apical_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_Ac_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "r_norm": [0.22653728723526, 0.22427527606487274, 0.2236478626728058]}, {"step": 2000, "epoch": 4, "train_loss": 0.10690974444150925, "cos_r_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_innovation_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_apical_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_Ac_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "r_norm": [0.2192738950252533, 0.21646450459957123, 0.21635594964027405], "ldr": {"dL_sdil": -0.004961423575878143, "dL_bp": -0.005419991910457611, "ratio": 0.9153931699243532}}, {"step": 2100, "epoch": 4, "train_loss": 0.04904601722955704, "cos_r_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_innovation_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_apical_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_Ac_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "r_norm": [0.25469183921813965, 0.24852119386196136, 0.2508085072040558]}, {"step": 2200, "epoch": 4, "train_loss": 0.07027833163738251, "cos_r_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_innovation_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_apical_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_Ac_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "r_norm": [0.22217589616775513, 0.21898141503334045, 0.2202150672674179]}, {"step": 2300, "epoch": 4, "train_loss": 0.1405230015516281, "cos_r_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_innovation_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_apical_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_Ac_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "r_norm": [0.2517160177230835, 0.24741223454475403, 0.2489166259765625]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9701, "test_loss": 0.10518000926971435}, {"step": 2400, "epoch": 5, "train_loss": 0.015669789165258408, "cos_r_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_innovation_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_apical_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_Ac_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "r_norm": [0.2449054718017578, 0.24151459336280823, 0.24172821640968323]}, {"step": 2500, "epoch": 5, "train_loss": 0.04790015518665314, "cos_r_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_innovation_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_apical_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_Ac_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "r_norm": [0.2145114541053772, 0.21184881031513214, 0.21148696541786194], "ldr": {"dL_sdil": -0.005982436239719391, "dL_bp": -0.0073370859026908875, "ratio": 0.8153695239584592}}, {"step": 2600, "epoch": 5, "train_loss": 0.014841018244624138, "cos_r_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_innovation_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_apical_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_Ac_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "r_norm": [0.19980891048908234, 0.19781577587127686, 0.19805651903152466]}, {"step": 2700, "epoch": 5, "train_loss": 0.07935170829296112, "cos_r_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_innovation_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_apical_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_Ac_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "r_norm": [0.23918147385120392, 0.23707768321037292, 0.2370166778564453]}, {"step": 2800, "epoch": 5, "train_loss": 0.005857329349964857, "cos_r_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_innovation_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_apical_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_Ac_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "r_norm": [0.21699143946170807, 0.2136717587709427, 0.21461525559425354]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.971, "test_loss": 0.10945739488601684}, {"step": 2900, "epoch": 6, "train_loss": 0.0123747568577528, "cos_r_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_innovation_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_apical_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_Ac_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "r_norm": [0.23866206407546997, 0.23430858552455902, 0.23588283360004425]}, {"step": 3000, "epoch": 6, "train_loss": 0.008246966637670994, "cos_r_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_innovation_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_apical_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_Ac_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "r_norm": [0.24846184253692627, 0.2451106160879135, 0.24654726684093475], "ldr": {"dL_sdil": -0.008115455508232117, "dL_bp": -0.010389789938926697, "ratio": 0.7810990940082926}}, {"step": 3100, "epoch": 6, "train_loss": 0.011437802575528622, "cos_r_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_innovation_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_apical_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_Ac_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "r_norm": [0.19606490433216095, 0.1932457983493805, 0.19307956099510193]}, {"step": 3200, "epoch": 6, "train_loss": 0.032173387706279755, "cos_r_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_innovation_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_apical_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_Ac_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "r_norm": [0.2475024312734604, 0.24428103864192963, 0.24396109580993652]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.971, "test_loss": 0.11738142681121826}, {"step": 3300, "epoch": 7, "train_loss": 0.0035855344031006098, "cos_r_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_innovation_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_apical_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_Ac_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "r_norm": [0.1960332989692688, 0.1931273192167282, 0.19294635951519012]}, {"step": 3400, "epoch": 7, "train_loss": 0.004453396890312433, "cos_r_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_innovation_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_apical_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_Ac_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "r_norm": [0.22042231261730194, 0.21685011684894562, 0.21672581136226654]}, {"step": 3500, "epoch": 7, "train_loss": 0.00869251973927021, "cos_r_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_innovation_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_apical_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_Ac_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "r_norm": [0.21137651801109314, 0.2083500772714615, 0.20858825743198395], "ldr": {"dL_sdil": -0.006606429815292358, "dL_bp": -0.009430110454559326, "ratio": 0.7005675964376686}}, {"step": 3600, "epoch": 7, "train_loss": 0.004775376059114933, "cos_r_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_innovation_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_apical_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_Ac_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "r_norm": [0.2073669135570526, 0.20420829951763153, 0.20337089896202087]}, {"step": 3700, "epoch": 7, "train_loss": 0.0054521383717656136, "cos_r_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_innovation_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_apical_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_Ac_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "r_norm": [0.20405349135398865, 0.20031914114952087, 0.20035898685455322]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9734, "test_loss": 0.11372013940811157}, {"step": 3800, "epoch": 8, "train_loss": 0.015979381278157234, "cos_r_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_innovation_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_apical_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_Ac_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "r_norm": [0.19954121112823486, 0.19605880975723267, 0.1963454782962799]}, {"step": 3900, "epoch": 8, "train_loss": 0.017817214131355286, "cos_r_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_innovation_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_apical_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_Ac_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "r_norm": [0.20743486285209656, 0.20354314148426056, 0.20332831144332886]}, {"step": 4000, "epoch": 8, "train_loss": 0.0004270627978257835, "cos_r_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_innovation_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_apical_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_Ac_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "r_norm": [0.19953644275665283, 0.19602517783641815, 0.1954791247844696], "ldr": {"dL_sdil": -0.008086413145065308, "dL_bp": -0.013367392122745514, "ratio": 0.6049357324758756}}, {"step": 4100, "epoch": 8, "train_loss": 0.07452463358640671, "cos_r_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_innovation_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_apical_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_Ac_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "r_norm": [0.20760531723499298, 0.20430520176887512, 0.20437899231910706]}, {"step": 4200, "epoch": 8, "train_loss": 0.03605803847312927, "cos_r_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_innovation_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_apical_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_Ac_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "r_norm": [0.19332198798656464, 0.1901654452085495, 0.18981432914733887]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9724, "test_loss": 0.11489360466003418}, {"step": 4300, "epoch": 9, "train_loss": 0.0038651630748063326, "cos_r_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_innovation_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_apical_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_Ac_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "r_norm": [0.21041682362556458, 0.20728406310081482, 0.20620422065258026]}, {"step": 4400, "epoch": 9, "train_loss": 0.0014019431546330452, "cos_r_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_innovation_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_apical_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_Ac_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "r_norm": [0.19830866158008575, 0.19524846971035004, 0.19488275051116943]}, {"step": 4500, "epoch": 9, "train_loss": 0.0025219277013093233, "cos_r_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_innovation_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_apical_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_Ac_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "r_norm": [0.19933810830116272, 0.195916086435318, 0.19609415531158447], "ldr": {"dL_sdil": -0.008369803428649902, "dL_bp": -0.014350831508636475, "ratio": 0.5832277679250063}}, {"step": 4600, "epoch": 9, "train_loss": 0.06718802452087402, "cos_r_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_innovation_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_apical_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_Ac_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "r_norm": [0.19856536388397217, 0.19568754732608795, 0.19562330842018127]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9747, "test_loss": 0.11986219282150269}, {"step": 4700, "epoch": 10, "train_loss": 0.0012022472219541669, "cos_r_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_innovation_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_apical_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_Ac_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "r_norm": [0.19958841800689697, 0.19681866466999054, 0.19656816124916077]}, {"step": 4800, "epoch": 10, "train_loss": 0.0013852894771844149, "cos_r_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_innovation_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_apical_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_Ac_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "r_norm": [0.19555793702602386, 0.1924840807914734, 0.1920878291130066]}, {"step": 4900, "epoch": 10, "train_loss": 0.0019364049658179283, "cos_r_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_innovation_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_apical_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_Ac_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "r_norm": [0.19706960022449493, 0.1942807137966156, 0.19380313158035278]}, {"step": 5000, "epoch": 10, "train_loss": 0.0011562240542843938, "cos_r_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_innovation_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_apical_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_Ac_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "r_norm": [0.19638454914093018, 0.19321104884147644, 0.1927313357591629], "ldr": {"dL_sdil": -0.008506037294864655, "dL_bp": -0.015625961124897003, "ratio": 0.544352902639179}}, {"step": 5100, "epoch": 10, "train_loss": 0.0011195843107998371, "cos_r_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_innovation_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_apical_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_Ac_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "r_norm": [0.21668797731399536, 0.21310555934906006, 0.21348032355308533]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9736, "test_loss": 0.12488943996429443}, {"step": 5200, "epoch": 11, "train_loss": 0.0005131271318532526, "cos_r_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_innovation_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_apical_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_Ac_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "r_norm": [0.2030562162399292, 0.1994873583316803, 0.19953997433185577]}, {"step": 5300, "epoch": 11, "train_loss": 0.0014590469654649496, "cos_r_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_innovation_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_apical_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_Ac_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "r_norm": [0.20533114671707153, 0.20210321247577667, 0.20187830924987793]}, {"step": 5400, "epoch": 11, "train_loss": 0.0009648285340517759, "cos_r_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_innovation_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_apical_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_Ac_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "r_norm": [0.2028876543045044, 0.19992581009864807, 0.19973811507225037]}, {"step": 5500, "epoch": 11, "train_loss": 0.002808267716318369, "cos_r_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_innovation_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_apical_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_Ac_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "r_norm": [0.19786036014556885, 0.19502316415309906, 0.19484545290470123], "ldr": {"dL_sdil": -0.009292647242546082, "dL_bp": -0.017724603414535522, "ratio": 0.5242795579237279}}, {"step": 5600, "epoch": 11, "train_loss": 0.0007443465292453766, "cos_r_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_innovation_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_apical_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_Ac_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "r_norm": [0.19685988128185272, 0.1940140724182129, 0.19382789731025696]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9741, "test_loss": 0.12913100547790526}, {"step": 5700, "epoch": 12, "train_loss": 0.00026087171863764524, "cos_r_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_innovation_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_apical_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_Ac_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "r_norm": [0.20257383584976196, 0.19969847798347473, 0.19937360286712646]}, {"step": 5800, "epoch": 12, "train_loss": 0.0011157530825585127, "cos_r_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_innovation_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_apical_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_Ac_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "r_norm": [0.20035849511623383, 0.1976122111082077, 0.1971856951713562]}, {"step": 5900, "epoch": 12, "train_loss": 0.0022946004755795, "cos_r_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_innovation_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_apical_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_Ac_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "r_norm": [0.20160531997680664, 0.19882169365882874, 0.1985234022140503]}, {"step": 6000, "epoch": 12, "train_loss": 0.0015348243759945035, "cos_r_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_innovation_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_apical_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_Ac_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "r_norm": [0.21117167174816132, 0.208292156457901, 0.20826399326324463], "ldr": {"dL_sdil": -0.010774381458759308, "dL_bp": -0.021835722029209137, "ratio": 0.49342913618091805}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9747, "test_loss": 0.13031967477798462}, {"step": 6100, "epoch": 13, "train_loss": 0.0005886715371161699, "cos_r_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_innovation_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_apical_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_Ac_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "r_norm": [0.20275822281837463, 0.1999770700931549, 0.1997852921485901]}, {"step": 6200, "epoch": 13, "train_loss": 0.00026136147789657116, "cos_r_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_innovation_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_apical_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_Ac_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "r_norm": [0.19926416873931885, 0.19659274816513062, 0.1963043063879013]}, {"step": 6300, "epoch": 13, "train_loss": 0.0008323256624862552, "cos_r_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_innovation_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_apical_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_Ac_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "r_norm": [0.19856512546539307, 0.19585520029067993, 0.19567559659481049]}, {"step": 6400, "epoch": 13, "train_loss": 0.0005636445712298155, "cos_r_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_innovation_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_apical_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_Ac_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "r_norm": [0.2014485001564026, 0.19853657484054565, 0.19835329055786133]}, {"step": 6500, "epoch": 13, "train_loss": 0.00021077357814647257, "cos_r_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_innovation_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_apical_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_Ac_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "r_norm": [0.19778192043304443, 0.19491931796073914, 0.19479045271873474], "ldr": {"dL_sdil": -0.010284215211868286, "dL_bp": -0.021059155464172363, "ratio": 0.4883488907883639}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9752, "test_loss": 0.13342348890304565}, {"step": 6600, "epoch": 14, "train_loss": 0.0004418176249600947, "cos_r_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_innovation_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_apical_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_Ac_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "r_norm": [0.20090188086032867, 0.19807308912277222, 0.19789955019950867]}, {"step": 6700, "epoch": 14, "train_loss": 0.001412956160493195, "cos_r_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_innovation_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_apical_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_Ac_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "r_norm": [0.2016269862651825, 0.19889488816261292, 0.19875365495681763]}, {"step": 6800, "epoch": 14, "train_loss": 0.0006913552642799914, "cos_r_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_innovation_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_apical_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_Ac_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "r_norm": [0.20099763572216034, 0.198038712143898, 0.19800257682800293]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006930493982508779, "cos_r_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_innovation_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_apical_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_Ac_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "r_norm": [0.19700448215007782, 0.19417372345924377, 0.19408582150936127]}, {"step": 7000, "epoch": 14, "train_loss": 0.00032816026941873133, "cos_r_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_innovation_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_apical_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_Ac_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "r_norm": [0.19693708419799805, 0.19410592317581177, 0.19396330416202545], "ldr": {"dL_sdil": -0.010280802845954895, "dL_bp": -0.021565861999988556, "ratio": 0.47671652753598953}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9753, "test_loss": 0.1358968108177185}], "final": {"test_acc": 0.9753, "test_loss": 0.1358968108177185, "wall_s": 29.24315071105957, "cos_r_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_innovation_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_apical_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_Ac_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_matched_s4.json b/results/nuis_ctrl_v1_rho0_matched_s4.json new file mode 100644 index 0000000..004e75b --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_matched_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_matched_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4173901081085205, "cos_r_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_innovation_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_apical_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_Ac_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "r_norm": [4.844976425170898, 4.853462219238281, 4.948795318603516], "ldr": {"dL_sdil": -0.00283050537109375, "dL_bp": -0.25957250595092773, "ratio": 0.01090448836530036}}, {"step": 100, "epoch": 0, "train_loss": 0.4020535349845886, "cos_r_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_innovation_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_apical_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_Ac_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "r_norm": [0.7995222806930542, 0.8062721490859985, 0.8229418396949768]}, {"step": 200, "epoch": 0, "train_loss": 0.2508821189403534, "cos_r_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_innovation_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_apical_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_Ac_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "r_norm": [0.7765392065048218, 0.7898226380348206, 0.8046278357505798]}, {"step": 300, "epoch": 0, "train_loss": 0.25044649839401245, "cos_r_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_innovation_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_apical_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_Ac_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "r_norm": [0.6173285841941833, 0.623796284198761, 0.6384584903717041]}, {"step": 400, "epoch": 0, "train_loss": 0.17270351946353912, "cos_r_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_innovation_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_apical_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_Ac_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "r_norm": [0.5093363523483276, 0.5141817331314087, 0.5286765098571777]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9455, "test_loss": 0.1889075668334961}, {"step": 500, "epoch": 1, "train_loss": 0.21437089145183563, "cos_r_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_innovation_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_apical_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_Ac_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "r_norm": [0.520584225654602, 0.5245216488838196, 0.5380268096923828], "ldr": {"dL_sdil": -0.012444496154785156, "dL_bp": -0.010713472962379456, "ratio": 1.161574421150286}}, {"step": 600, "epoch": 1, "train_loss": 0.22007516026496887, "cos_r_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_innovation_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_apical_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_Ac_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "r_norm": [0.5072118639945984, 0.507872462272644, 0.5214201211929321]}, {"step": 700, "epoch": 1, "train_loss": 0.10645534098148346, "cos_r_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_innovation_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_apical_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_Ac_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "r_norm": [0.3919033408164978, 0.3957861661911011, 0.4059203267097473]}, {"step": 800, "epoch": 1, "train_loss": 0.099101722240448, "cos_r_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_innovation_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_apical_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_Ac_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "r_norm": [0.35605648159980774, 0.3594273328781128, 0.36916738748550415]}, {"step": 900, "epoch": 1, "train_loss": 0.11939629912376404, "cos_r_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_innovation_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_apical_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_Ac_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "r_norm": [0.35462111234664917, 0.356706440448761, 0.3648630678653717]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9565, "test_loss": 0.1442838306427002}, {"step": 1000, "epoch": 2, "train_loss": 0.10045582056045532, "cos_r_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_innovation_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_apical_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_Ac_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "r_norm": [0.28615742921829224, 0.288568913936615, 0.29509449005126953], "ldr": {"dL_sdil": -0.0052221789956092834, "dL_bp": -0.004839010536670685, "ratio": 1.0791832247594617}}, {"step": 1100, "epoch": 2, "train_loss": 0.09472673386335373, "cos_r_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_innovation_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_apical_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_Ac_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "r_norm": [0.29523903131484985, 0.295265257358551, 0.30285394191741943]}, {"step": 1200, "epoch": 2, "train_loss": 0.02559100277721882, "cos_r_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_innovation_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_apical_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_Ac_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "r_norm": [0.30911892652511597, 0.31053924560546875, 0.3160984516143799]}, {"step": 1300, "epoch": 2, "train_loss": 0.15685486793518066, "cos_r_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_innovation_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_apical_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_Ac_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "r_norm": [0.2874014675617218, 0.290073037147522, 0.2968340814113617]}, {"step": 1400, "epoch": 2, "train_loss": 0.23597419261932373, "cos_r_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_innovation_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_apical_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_Ac_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "r_norm": [0.3454799950122833, 0.34709277749061584, 0.35570353269577026]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9584, "test_loss": 0.1362682201385498}, {"step": 1500, "epoch": 3, "train_loss": 0.049877192825078964, "cos_r_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_innovation_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_apical_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_Ac_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "r_norm": [0.32026493549346924, 0.32111209630966187, 0.32673612236976624], "ldr": {"dL_sdil": -0.008508473634719849, "dL_bp": -0.007917135953903198, "ratio": 1.0746908584442227}}, {"step": 1600, "epoch": 3, "train_loss": 0.04681463539600372, "cos_r_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_innovation_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_apical_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_Ac_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "r_norm": [0.2679218351840973, 0.2675299048423767, 0.2748056650161743]}, {"step": 1700, "epoch": 3, "train_loss": 0.042607326060533524, "cos_r_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_innovation_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_apical_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_Ac_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "r_norm": [0.25324034690856934, 0.25399982929229736, 0.25864163041114807]}, {"step": 1800, "epoch": 3, "train_loss": 0.05712303891777992, "cos_r_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_innovation_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_apical_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_Ac_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "r_norm": [0.26621386408805847, 0.2666720151901245, 0.27278897166252136]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9686, "test_loss": 0.10741588859558106}, {"step": 1900, "epoch": 4, "train_loss": 0.0610680989921093, "cos_r_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_innovation_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_apical_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_Ac_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "r_norm": [0.255119651556015, 0.25568583607673645, 0.26110944151878357]}, {"step": 2000, "epoch": 4, "train_loss": 0.059706609696149826, "cos_r_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_innovation_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_apical_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_Ac_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "r_norm": [0.21822944283485413, 0.2185012400150299, 0.22326743602752686], "ldr": {"dL_sdil": -0.004507958889007568, "dL_bp": -0.004968710243701935, "ratio": 0.9072694256465469}}, {"step": 2100, "epoch": 4, "train_loss": 0.032644227147102356, "cos_r_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_innovation_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_apical_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_Ac_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "r_norm": [0.24712875485420227, 0.24843883514404297, 0.2534225881099701]}, {"step": 2200, "epoch": 4, "train_loss": 0.028950463980436325, "cos_r_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_innovation_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_apical_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_Ac_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "r_norm": [0.24966076016426086, 0.2504446804523468, 0.25478261709213257]}, {"step": 2300, "epoch": 4, "train_loss": 0.0650198757648468, "cos_r_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_innovation_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_apical_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_Ac_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "r_norm": [0.20470218360424042, 0.2031494677066803, 0.20760329067707062]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9693, "test_loss": 0.10958373928070068}, {"step": 2400, "epoch": 5, "train_loss": 0.03290189430117607, "cos_r_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_innovation_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_apical_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_Ac_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "r_norm": [0.21331188082695007, 0.21278373897075653, 0.2184755653142929]}, {"step": 2500, "epoch": 5, "train_loss": 0.03874827176332474, "cos_r_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_innovation_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_apical_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_Ac_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "r_norm": [0.22789107263088226, 0.2283928245306015, 0.23281589150428772], "ldr": {"dL_sdil": -0.0070598796010017395, "dL_bp": -0.008158765733242035, "ratio": 0.8653122092030416}}, {"step": 2600, "epoch": 5, "train_loss": 0.021128688007593155, "cos_r_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_innovation_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_apical_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_Ac_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "r_norm": [0.2212328165769577, 0.2204180359840393, 0.2251305729150772]}, {"step": 2700, "epoch": 5, "train_loss": 0.07236946374177933, "cos_r_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_innovation_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_apical_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_Ac_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "r_norm": [0.2149030566215515, 0.2140052616596222, 0.2162187099456787]}, {"step": 2800, "epoch": 5, "train_loss": 0.008522935211658478, "cos_r_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_innovation_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_apical_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_Ac_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "r_norm": [0.205214723944664, 0.2052396833896637, 0.2100837081670761]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9724, "test_loss": 0.10797543697357177}, {"step": 2900, "epoch": 6, "train_loss": 0.018756665289402008, "cos_r_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_innovation_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_apical_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_Ac_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "r_norm": [0.22938458621501923, 0.22977539896965027, 0.23387743532657623]}, {"step": 3000, "epoch": 6, "train_loss": 0.005393678322434425, "cos_r_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_innovation_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_apical_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_Ac_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "r_norm": [0.19885726273059845, 0.1973133385181427, 0.20168818533420563], "ldr": {"dL_sdil": -0.005929701030254364, "dL_bp": -0.008110091090202332, "ratio": 0.7311509777514015}}, {"step": 3100, "epoch": 6, "train_loss": 0.03258326277136803, "cos_r_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_innovation_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_apical_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_Ac_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "r_norm": [0.22770541906356812, 0.22627578675746918, 0.22949440777301788]}, {"step": 3200, "epoch": 6, "train_loss": 0.05724141746759415, "cos_r_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_innovation_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_apical_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_Ac_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "r_norm": [0.23419582843780518, 0.23333817720413208, 0.23697063326835632]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9735, "test_loss": 0.1085232892036438}, {"step": 3300, "epoch": 7, "train_loss": 0.044839728623628616, "cos_r_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_innovation_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_apical_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_Ac_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "r_norm": [0.18712347745895386, 0.185409277677536, 0.1905364990234375]}, {"step": 3400, "epoch": 7, "train_loss": 0.008097128942608833, "cos_r_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_innovation_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_apical_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_Ac_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "r_norm": [0.19379451870918274, 0.19184225797653198, 0.19568520784378052]}, {"step": 3500, "epoch": 7, "train_loss": 0.010430501773953438, "cos_r_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_innovation_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_apical_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_Ac_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "r_norm": [0.1839253008365631, 0.18301129341125488, 0.18623384833335876], "ldr": {"dL_sdil": -0.006262347102165222, "dL_bp": -0.009936518967151642, "ratio": 0.6302355103298675}}, {"step": 3600, "epoch": 7, "train_loss": 0.0012780515244230628, "cos_r_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_innovation_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_apical_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_Ac_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "r_norm": [0.1790696531534195, 0.1764722466468811, 0.17974436283111572]}, {"step": 3700, "epoch": 7, "train_loss": 0.018456075340509415, "cos_r_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_innovation_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_apical_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_Ac_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "r_norm": [0.18262530863285065, 0.18070702254772186, 0.1839236468076706]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.973, "test_loss": 0.11429949645996093}, {"step": 3800, "epoch": 8, "train_loss": 0.019817866384983063, "cos_r_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_innovation_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_apical_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_Ac_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "r_norm": [0.1908964067697525, 0.19014066457748413, 0.19408388435840607]}, {"step": 3900, "epoch": 8, "train_loss": 0.019774047657847404, "cos_r_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_innovation_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_apical_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_Ac_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "r_norm": [0.19253888726234436, 0.19145795702934265, 0.19445383548736572]}, {"step": 4000, "epoch": 8, "train_loss": 0.0010947450064122677, "cos_r_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_innovation_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_apical_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_Ac_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "r_norm": [0.18950730562210083, 0.18815365433692932, 0.19119828939437866], "ldr": {"dL_sdil": -0.007813781499862671, "dL_bp": -0.014065638184547424, "ratio": 0.5555227141024378}}, {"step": 4100, "epoch": 8, "train_loss": 0.09640847891569138, "cos_r_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_innovation_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_apical_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_Ac_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "r_norm": [0.1922830492258072, 0.19028913974761963, 0.19349952042102814]}, {"step": 4200, "epoch": 8, "train_loss": 0.014551833271980286, "cos_r_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_innovation_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_apical_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_Ac_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "r_norm": [0.19427847862243652, 0.19271467626094818, 0.19574645161628723]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9729, "test_loss": 0.12127476367950439}, {"step": 4300, "epoch": 9, "train_loss": 0.0006672337185591459, "cos_r_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_innovation_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_apical_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_Ac_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "r_norm": [0.19403856992721558, 0.19274917244911194, 0.1965358704328537]}, {"step": 4400, "epoch": 9, "train_loss": 0.004814686719328165, "cos_r_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_innovation_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_apical_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_Ac_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "r_norm": [0.1811094731092453, 0.1802864819765091, 0.18380987644195557]}, {"step": 4500, "epoch": 9, "train_loss": 0.007938915863633156, "cos_r_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_innovation_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_apical_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_Ac_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "r_norm": [0.18736639618873596, 0.185921311378479, 0.1901116669178009], "ldr": {"dL_sdil": -0.008901968598365784, "dL_bp": -0.017541170120239258, "ratio": 0.5074900099221181}}, {"step": 4600, "epoch": 9, "train_loss": 0.0795854851603508, "cos_r_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_innovation_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_apical_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_Ac_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "r_norm": [0.184382826089859, 0.1827830672264099, 0.18596649169921875]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9742, "test_loss": 0.12309427490234374}, {"step": 4700, "epoch": 10, "train_loss": 0.0007766007911413908, "cos_r_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_innovation_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_apical_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_Ac_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "r_norm": [0.19498443603515625, 0.19344991445541382, 0.19718916714191437]}, {"step": 4800, "epoch": 10, "train_loss": 0.002889199648052454, "cos_r_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_innovation_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_apical_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_Ac_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "r_norm": [0.19870562851428986, 0.1971784234046936, 0.20026731491088867]}, {"step": 4900, "epoch": 10, "train_loss": 0.0008581852307543159, "cos_r_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_innovation_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_apical_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_Ac_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "r_norm": [0.19465115666389465, 0.19327929615974426, 0.19643467664718628]}, {"step": 5000, "epoch": 10, "train_loss": 0.0007646966842003167, "cos_r_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_innovation_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_apical_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_Ac_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "r_norm": [0.19242608547210693, 0.19134974479675293, 0.19527661800384521], "ldr": {"dL_sdil": -0.009105280041694641, "dL_bp": -0.018391385674476624, "ratio": 0.4950839595697705}}, {"step": 5100, "epoch": 10, "train_loss": 0.0015073801623657346, "cos_r_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_innovation_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_apical_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_Ac_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "r_norm": [0.17740818858146667, 0.17602333426475525, 0.17976273596286774]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9745, "test_loss": 0.12885003728866576}, {"step": 5200, "epoch": 11, "train_loss": 0.0014642335008829832, "cos_r_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_innovation_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_apical_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_Ac_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "r_norm": [0.17490693926811218, 0.17364893853664398, 0.17717021703720093]}, {"step": 5300, "epoch": 11, "train_loss": 0.001281333388760686, "cos_r_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_innovation_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_apical_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_Ac_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "r_norm": [0.1789165437221527, 0.1773441880941391, 0.1806153953075409]}, {"step": 5400, "epoch": 11, "train_loss": 0.0045455265790224075, "cos_r_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_innovation_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_apical_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_Ac_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "r_norm": [0.18679404258728027, 0.18510667979717255, 0.18804368376731873]}, {"step": 5500, "epoch": 11, "train_loss": 0.002247859025374055, "cos_r_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_innovation_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_apical_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_Ac_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "r_norm": [0.1849762499332428, 0.18319714069366455, 0.18662986159324646], "ldr": {"dL_sdil": -0.00928555428981781, "dL_bp": -0.02012455463409424, "ratio": 0.4614042128458627}}, {"step": 5600, "epoch": 11, "train_loss": 0.0012828883482143283, "cos_r_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_innovation_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_apical_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_Ac_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "r_norm": [0.1720443069934845, 0.1705038845539093, 0.17383548617362976]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9739, "test_loss": 0.1312964542388916}, {"step": 5700, "epoch": 12, "train_loss": 0.0005914976936765015, "cos_r_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_innovation_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_apical_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_Ac_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "r_norm": [0.18014508485794067, 0.1789059042930603, 0.18226537108421326]}, {"step": 5800, "epoch": 12, "train_loss": 0.0010177994845435023, "cos_r_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_innovation_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_apical_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_Ac_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "r_norm": [0.19342085719108582, 0.19208499789237976, 0.19565963745117188]}, {"step": 5900, "epoch": 12, "train_loss": 0.0026742268819361925, "cos_r_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_innovation_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_apical_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_Ac_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "r_norm": [0.18534237146377563, 0.18388569355010986, 0.18735302984714508]}, {"step": 6000, "epoch": 12, "train_loss": 0.0006315991631709039, "cos_r_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_innovation_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_apical_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_Ac_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "r_norm": [0.18172697722911835, 0.1802336871623993, 0.18384242057800293], "ldr": {"dL_sdil": -0.009645640850067139, "dL_bp": -0.021210603415966034, "ratio": 0.4547556078865015}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9741, "test_loss": 0.13412653560638427}, {"step": 6100, "epoch": 13, "train_loss": 0.0009790371404960752, "cos_r_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_innovation_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_apical_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_Ac_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "r_norm": [0.18465885519981384, 0.18316440284252167, 0.18671941757202148]}, {"step": 6200, "epoch": 13, "train_loss": 0.0008633113466203213, "cos_r_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_innovation_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_apical_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_Ac_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "r_norm": [0.18658879399299622, 0.18528807163238525, 0.18887931108474731]}, {"step": 6300, "epoch": 13, "train_loss": 0.0002778768539428711, "cos_r_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_innovation_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_apical_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_Ac_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "r_norm": [0.18958455324172974, 0.188012033700943, 0.19160720705986023]}, {"step": 6400, "epoch": 13, "train_loss": 0.00041367538506165147, "cos_r_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_innovation_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_apical_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_Ac_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "r_norm": [0.19554319977760315, 0.1940520703792572, 0.19749662280082703]}, {"step": 6500, "epoch": 13, "train_loss": 0.00021037989063188434, "cos_r_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_innovation_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_apical_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_Ac_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "r_norm": [0.18189536035060883, 0.18059924244880676, 0.18395429849624634], "ldr": {"dL_sdil": -0.009790733456611633, "dL_bp": -0.022421933710575104, "ratio": 0.43665874598469273}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9738, "test_loss": 0.1366198040008545}, {"step": 6600, "epoch": 14, "train_loss": 0.0006715196650475264, "cos_r_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_innovation_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_apical_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_Ac_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "r_norm": [0.17912247776985168, 0.17778383195400238, 0.18115678429603577]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008008020813576877, "cos_r_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_innovation_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_apical_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_Ac_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "r_norm": [0.17753815650939941, 0.17617246508598328, 0.1796029806137085]}, {"step": 6800, "epoch": 14, "train_loss": 0.0007890297565609217, "cos_r_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_innovation_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_apical_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_Ac_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "r_norm": [0.18021561205387115, 0.1791214942932129, 0.18264533579349518]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005114716477692127, "cos_r_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_innovation_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_apical_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_Ac_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "r_norm": [0.17889344692230225, 0.17748130857944489, 0.18085458874702454]}, {"step": 7000, "epoch": 14, "train_loss": 0.0008774346788413823, "cos_r_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_innovation_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_apical_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_Ac_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "r_norm": [0.18395139276981354, 0.18258227407932281, 0.18588440120220184], "ldr": {"dL_sdil": -0.010152354836463928, "dL_bp": -0.023831740021705627, "ratio": 0.4260014093480921}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9743, "test_loss": 0.13829146881103516}], "final": {"test_acc": 0.9743, "test_loss": 0.13829146881103516, "wall_s": 23.967039346694946, "cos_r_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_innovation_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_apical_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_Ac_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_raw_s0.json b/results/nuis_ctrl_v1_rho0_raw_s0.json new file mode 100644 index 0000000..191e4f6 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_raw_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_raw_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3978137969970703, "cos_r_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_innovation_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_apical_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_Ac_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "r_norm": [4.8918256759643555, 4.819490432739258, 4.813833236694336], "ldr": {"dL_sdil": -0.007525444030761719, "dL_bp": -0.23199224472045898, "ratio": 0.03243834309991511}}, {"step": 100, "epoch": 0, "train_loss": 0.33160343766212463, "cos_r_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_innovation_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_apical_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_Ac_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "r_norm": [0.8375285267829895, 0.818266749382019, 0.8186335563659668]}, {"step": 200, "epoch": 0, "train_loss": 0.29867634177207947, "cos_r_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_innovation_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_apical_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_Ac_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "r_norm": [0.7242861390113831, 0.7113809585571289, 0.7089210152626038]}, {"step": 300, "epoch": 0, "train_loss": 0.28106310963630676, "cos_r_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_innovation_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_apical_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_Ac_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "r_norm": [0.5968752503395081, 0.5882158875465393, 0.586020827293396]}, {"step": 400, "epoch": 0, "train_loss": 0.13037653267383575, "cos_r_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_innovation_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_apical_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_Ac_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "r_norm": [0.5326493978500366, 0.5187801718711853, 0.5172048807144165]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9438, "test_loss": 0.19445072021484375}, {"step": 500, "epoch": 1, "train_loss": 0.3059273362159729, "cos_r_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_innovation_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_apical_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_Ac_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "r_norm": [0.5493181943893433, 0.5408297777175903, 0.5343528985977173], "ldr": {"dL_sdil": -0.017139852046966553, "dL_bp": -0.0144614577293396, "ratio": 1.1852091516467935}}, {"step": 600, "epoch": 1, "train_loss": 0.2938694357872009, "cos_r_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_innovation_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_apical_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_Ac_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "r_norm": [0.5047881007194519, 0.4929560720920563, 0.4911949336528778]}, {"step": 700, "epoch": 1, "train_loss": 0.0692189484834671, "cos_r_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_innovation_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_apical_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_Ac_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "r_norm": [0.4124377369880676, 0.4063720703125, 0.40228134393692017]}, {"step": 800, "epoch": 1, "train_loss": 0.10150999575853348, "cos_r_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_innovation_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_apical_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_Ac_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "r_norm": [0.41368532180786133, 0.408090204000473, 0.40460047125816345]}, {"step": 900, "epoch": 1, "train_loss": 0.08599556982517242, "cos_r_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_innovation_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_apical_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_Ac_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "r_norm": [0.35195469856262207, 0.34207263588905334, 0.34272995591163635]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9534, "test_loss": 0.15320639877319336}, {"step": 1000, "epoch": 2, "train_loss": 0.19156473875045776, "cos_r_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_innovation_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_apical_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_Ac_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "r_norm": [0.360003262758255, 0.3540140390396118, 0.3526112735271454], "ldr": {"dL_sdil": -0.009464025497436523, "dL_bp": -0.008196189999580383, "ratio": 1.1546859574901325}}, {"step": 1100, "epoch": 2, "train_loss": 0.10864664614200592, "cos_r_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_innovation_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_apical_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_Ac_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "r_norm": [0.28866949677467346, 0.2821740508079529, 0.28306305408477783]}, {"step": 1200, "epoch": 2, "train_loss": 0.031033102422952652, "cos_r_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_innovation_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_apical_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_Ac_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "r_norm": [0.30560463666915894, 0.30166518688201904, 0.29685312509536743]}, {"step": 1300, "epoch": 2, "train_loss": 0.15016421675682068, "cos_r_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_innovation_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_apical_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_Ac_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "r_norm": [0.279674768447876, 0.27491551637649536, 0.27148789167404175]}, {"step": 1400, "epoch": 2, "train_loss": 0.22877365350723267, "cos_r_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_innovation_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_apical_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_Ac_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "r_norm": [0.29575562477111816, 0.2879781126976013, 0.28471633791923523]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9592, "test_loss": 0.13688001098632813}, {"step": 1500, "epoch": 3, "train_loss": 0.042368389666080475, "cos_r_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_innovation_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_apical_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_Ac_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "r_norm": [0.3197419345378876, 0.3114323019981384, 0.3098006248474121], "ldr": {"dL_sdil": -0.008259907364845276, "dL_bp": -0.007200941443443298, "ratio": 1.1470593712945967}}, {"step": 1600, "epoch": 3, "train_loss": 0.05860002711415291, "cos_r_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_innovation_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_apical_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_Ac_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "r_norm": [0.30268725752830505, 0.29540443420410156, 0.2939055562019348]}, {"step": 1700, "epoch": 3, "train_loss": 0.08397884666919708, "cos_r_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_innovation_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_apical_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_Ac_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "r_norm": [0.29046741127967834, 0.2826988697052002, 0.27999138832092285]}, {"step": 1800, "epoch": 3, "train_loss": 0.10174758732318878, "cos_r_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_innovation_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_apical_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_Ac_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "r_norm": [0.30012351274490356, 0.29224371910095215, 0.28972357511520386]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9665, "test_loss": 0.1125657865524292}, {"step": 1900, "epoch": 4, "train_loss": 0.07242363691329956, "cos_r_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_innovation_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_apical_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_Ac_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "r_norm": [0.28996407985687256, 0.28326937556266785, 0.2812224328517914]}, {"step": 2000, "epoch": 4, "train_loss": 0.06877899169921875, "cos_r_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_innovation_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_apical_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_Ac_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "r_norm": [0.245533287525177, 0.24014459550380707, 0.23770779371261597], "ldr": {"dL_sdil": -0.006018199026584625, "dL_bp": -0.006462790071964264, "ratio": 0.9312075681820016}}, {"step": 2100, "epoch": 4, "train_loss": 0.05175407975912094, "cos_r_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_innovation_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_apical_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_Ac_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "r_norm": [0.24031473696231842, 0.23528200387954712, 0.23227009177207947]}, {"step": 2200, "epoch": 4, "train_loss": 0.04244621843099594, "cos_r_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_innovation_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_apical_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_Ac_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "r_norm": [0.2526210844516754, 0.24659496545791626, 0.24442610144615173]}, {"step": 2300, "epoch": 4, "train_loss": 0.12019313126802444, "cos_r_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_innovation_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_apical_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_Ac_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "r_norm": [0.21065500378608704, 0.20645348727703094, 0.20339635014533997]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9679, "test_loss": 0.11181112232208253}, {"step": 2400, "epoch": 5, "train_loss": 0.033473387360572815, "cos_r_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_innovation_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_apical_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_Ac_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "r_norm": [0.22075463831424713, 0.2167082130908966, 0.21338585019111633]}, {"step": 2500, "epoch": 5, "train_loss": 0.046647246927022934, "cos_r_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_innovation_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_apical_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_Ac_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "r_norm": [0.23498451709747314, 0.2304421216249466, 0.22597557306289673], "ldr": {"dL_sdil": -0.00819263607263565, "dL_bp": -0.009024456143379211, "ratio": 0.9078260166011416}}, {"step": 2600, "epoch": 5, "train_loss": 0.023400278761982918, "cos_r_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_innovation_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_apical_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_Ac_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "r_norm": [0.2576431632041931, 0.2516178488731384, 0.24756699800491333]}, {"step": 2700, "epoch": 5, "train_loss": 0.05335155129432678, "cos_r_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_innovation_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_apical_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_Ac_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "r_norm": [0.2206578552722931, 0.2165922373533249, 0.2133471816778183]}, {"step": 2800, "epoch": 5, "train_loss": 0.020739704370498657, "cos_r_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_innovation_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_apical_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_Ac_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "r_norm": [0.2329060584306717, 0.2280796319246292, 0.22505390644073486]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9685, "test_loss": 0.11429850444793702}, {"step": 2900, "epoch": 6, "train_loss": 0.02710738405585289, "cos_r_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_innovation_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_apical_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_Ac_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "r_norm": [0.2212202548980713, 0.21703368425369263, 0.21472647786140442]}, {"step": 3000, "epoch": 6, "train_loss": 0.003210586030036211, "cos_r_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_innovation_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_apical_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_Ac_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "r_norm": [0.19350673258304596, 0.1897171288728714, 0.18670257925987244], "ldr": {"dL_sdil": -0.007917709648609161, "dL_bp": -0.009828649461269379, "ratio": 0.805574527793423}}, {"step": 3100, "epoch": 6, "train_loss": 0.06387314200401306, "cos_r_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_innovation_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_apical_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_Ac_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "r_norm": [0.17050178349018097, 0.16748245060443878, 0.16528522968292236]}, {"step": 3200, "epoch": 6, "train_loss": 0.04849023371934891, "cos_r_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_innovation_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_apical_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_Ac_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "r_norm": [0.18880192935466766, 0.18442402780056, 0.18181553483009338]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9723, "test_loss": 0.11568929290771485}, {"step": 3300, "epoch": 7, "train_loss": 0.006756230723112822, "cos_r_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_innovation_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_apical_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_Ac_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "r_norm": [0.1959608942270279, 0.19186583161354065, 0.1897565722465515]}, {"step": 3400, "epoch": 7, "train_loss": 0.005082777701318264, "cos_r_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_innovation_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_apical_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_Ac_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "r_norm": [0.2011243999004364, 0.19720213115215302, 0.19440439343452454]}, {"step": 3500, "epoch": 7, "train_loss": 0.008085569366812706, "cos_r_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_innovation_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_apical_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_Ac_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "r_norm": [0.1824260950088501, 0.17886941134929657, 0.17662227153778076], "ldr": {"dL_sdil": -0.008172653615474701, "dL_bp": -0.011752061545848846, "ratio": 0.6954229760957564}}, {"step": 3600, "epoch": 7, "train_loss": 0.009758161380887032, "cos_r_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_innovation_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_apical_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_Ac_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "r_norm": [0.15041005611419678, 0.1479288935661316, 0.1454424113035202]}, {"step": 3700, "epoch": 7, "train_loss": 0.005388827063143253, "cos_r_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_innovation_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_apical_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_Ac_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "r_norm": [0.1920110434293747, 0.18857887387275696, 0.185125932097435]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9706, "test_loss": 0.12330467185974121}, {"step": 3800, "epoch": 8, "train_loss": 0.056129325181245804, "cos_r_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_innovation_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_apical_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_Ac_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "r_norm": [0.1895904839038849, 0.18607723712921143, 0.18282277882099152]}, {"step": 3900, "epoch": 8, "train_loss": 0.011785263195633888, "cos_r_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_innovation_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_apical_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_Ac_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "r_norm": [0.17273201048374176, 0.16941824555397034, 0.1666167974472046]}, {"step": 4000, "epoch": 8, "train_loss": 0.000689225154928863, "cos_r_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_innovation_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_apical_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_Ac_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "r_norm": [0.17630384862422943, 0.17299006879329681, 0.17047850787639618], "ldr": {"dL_sdil": -0.009058646857738495, "dL_bp": -0.014096729457378387, "ratio": 0.6426062786497684}}, {"step": 4100, "epoch": 8, "train_loss": 0.03716294467449188, "cos_r_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_innovation_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_apical_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_Ac_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "r_norm": [0.1756715625524521, 0.17255355417728424, 0.16945818066596985]}, {"step": 4200, "epoch": 8, "train_loss": 0.007642740849405527, "cos_r_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_innovation_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_apical_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_Ac_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "r_norm": [0.17260634899139404, 0.16935701668262482, 0.1664430946111679]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9715, "test_loss": 0.12795383539199828}, {"step": 4300, "epoch": 9, "train_loss": 0.005126616917550564, "cos_r_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_innovation_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_apical_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_Ac_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "r_norm": [0.19533851742744446, 0.19135543704032898, 0.1880948841571808]}, {"step": 4400, "epoch": 9, "train_loss": 0.003296743845567107, "cos_r_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_innovation_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_apical_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_Ac_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "r_norm": [0.18917936086654663, 0.1854909360408783, 0.18246984481811523]}, {"step": 4500, "epoch": 9, "train_loss": 0.0014693234115839005, "cos_r_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_innovation_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_apical_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_Ac_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "r_norm": [0.17914845049381256, 0.17544743418693542, 0.17231427133083344], "ldr": {"dL_sdil": -0.00979585200548172, "dL_bp": -0.01655876636505127, "ratio": 0.5915810266009143}}, {"step": 4600, "epoch": 9, "train_loss": 0.032596148550510406, "cos_r_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_innovation_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_apical_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_Ac_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "r_norm": [0.19013909995555878, 0.1861753761768341, 0.1826411783695221]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.974, "test_loss": 0.12575695457458497}, {"step": 4700, "epoch": 10, "train_loss": 0.0034915334545075893, "cos_r_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_innovation_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_apical_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_Ac_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "r_norm": [0.18801575899124146, 0.1843785047531128, 0.18106210231781006]}, {"step": 4800, "epoch": 10, "train_loss": 0.0008336055907420814, "cos_r_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_innovation_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_apical_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_Ac_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "r_norm": [0.1824166178703308, 0.17903029918670654, 0.17574921250343323]}, {"step": 4900, "epoch": 10, "train_loss": 0.001636673929169774, "cos_r_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_innovation_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_apical_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_Ac_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "r_norm": [0.17372174561023712, 0.17035624384880066, 0.16707497835159302]}, {"step": 5000, "epoch": 10, "train_loss": 0.0015556986909359694, "cos_r_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_innovation_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_apical_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_Ac_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "r_norm": [0.17774055898189545, 0.1746811866760254, 0.1715075671672821], "ldr": {"dL_sdil": -0.010479673743247986, "dL_bp": -0.01817304641008377, "ratio": 0.5766602641499378}}, {"step": 5100, "epoch": 10, "train_loss": 0.0015227128751575947, "cos_r_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_innovation_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_apical_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_Ac_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "r_norm": [0.18908870220184326, 0.18590903282165527, 0.18229123950004578]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9733, "test_loss": 0.12908906383514404}, {"step": 5200, "epoch": 11, "train_loss": 0.001309263869188726, "cos_r_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_innovation_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_apical_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_Ac_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "r_norm": [0.17493118345737457, 0.17182868719100952, 0.16859647631645203]}, {"step": 5300, "epoch": 11, "train_loss": 0.0011599198915064335, "cos_r_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_innovation_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_apical_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_Ac_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "r_norm": [0.16871681809425354, 0.16564862430095673, 0.16264042258262634]}, {"step": 5400, "epoch": 11, "train_loss": 0.0014726945664733648, "cos_r_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_innovation_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_apical_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_Ac_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "r_norm": [0.1738046109676361, 0.17061923444271088, 0.167450949549675]}, {"step": 5500, "epoch": 11, "train_loss": 0.0022923252545297146, "cos_r_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_innovation_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_apical_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_Ac_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "r_norm": [0.1730632185935974, 0.1701216697692871, 0.16676318645477295], "ldr": {"dL_sdil": -0.01092517375946045, "dL_bp": -0.019617542624473572, "ratio": 0.5569083737241846}}, {"step": 5600, "epoch": 11, "train_loss": 0.003013922367244959, "cos_r_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_innovation_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_apical_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_Ac_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "r_norm": [0.17643772065639496, 0.17336799204349518, 0.16997528076171875]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.974, "test_loss": 0.1334169048309326}, {"step": 5700, "epoch": 12, "train_loss": 0.00026223674649372697, "cos_r_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_innovation_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_apical_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_Ac_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "r_norm": [0.1758565604686737, 0.17264872789382935, 0.16945300996303558]}, {"step": 5800, "epoch": 12, "train_loss": 0.0003017765120603144, "cos_r_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_innovation_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_apical_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_Ac_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "r_norm": [0.17203187942504883, 0.1690245270729065, 0.16582567989826202]}, {"step": 5900, "epoch": 12, "train_loss": 0.0013039933983236551, "cos_r_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_innovation_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_apical_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_Ac_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "r_norm": [0.17366287112236023, 0.17064981162548065, 0.16749976575374603]}, {"step": 6000, "epoch": 12, "train_loss": 0.0008040090324357152, "cos_r_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_innovation_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_apical_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_Ac_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "r_norm": [0.17631205916404724, 0.17321062088012695, 0.17003434896469116], "ldr": {"dL_sdil": -0.011585041880607605, "dL_bp": -0.022001802921295166, "ratio": 0.526549661500451}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9745, "test_loss": 0.1352493793487549}, {"step": 6100, "epoch": 13, "train_loss": 0.0014463819097727537, "cos_r_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_innovation_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_apical_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_Ac_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "r_norm": [0.17908143997192383, 0.17593857645988464, 0.17247158288955688]}, {"step": 6200, "epoch": 13, "train_loss": 0.0002795365871861577, "cos_r_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_innovation_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_apical_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_Ac_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "r_norm": [0.17346340417861938, 0.17047497630119324, 0.16715440154075623]}, {"step": 6300, "epoch": 13, "train_loss": 0.0003245513653382659, "cos_r_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_innovation_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_apical_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_Ac_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "r_norm": [0.1769692301750183, 0.17405544221401215, 0.17055493593215942]}, {"step": 6400, "epoch": 13, "train_loss": 0.0006700620288029313, "cos_r_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_innovation_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_apical_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_Ac_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "r_norm": [0.17150437831878662, 0.16849254071712494, 0.16540826857089996]}, {"step": 6500, "epoch": 13, "train_loss": 0.00013033303548581898, "cos_r_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_innovation_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_apical_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_Ac_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "r_norm": [0.16887250542640686, 0.1660538762807846, 0.16295760869979858], "ldr": {"dL_sdil": -0.011667512357234955, "dL_bp": -0.022730067372322083, "ratio": 0.5133074251879357}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9737, "test_loss": 0.13770540838241577}, {"step": 6600, "epoch": 14, "train_loss": 0.002334980759769678, "cos_r_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_innovation_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_apical_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_Ac_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "r_norm": [0.1712503284215927, 0.16834032535552979, 0.16512611508369446]}, {"step": 6700, "epoch": 14, "train_loss": 0.0007730001234449446, "cos_r_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_innovation_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_apical_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_Ac_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "r_norm": [0.17286744713783264, 0.16990327835083008, 0.1666618287563324]}, {"step": 6800, "epoch": 14, "train_loss": 0.0018118484877049923, "cos_r_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_innovation_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_apical_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_Ac_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "r_norm": [0.17235802114009857, 0.16948559880256653, 0.166168212890625]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005928482860326767, "cos_r_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_innovation_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_apical_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_Ac_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "r_norm": [0.17221295833587646, 0.16921906173229218, 0.16597718000411987]}, {"step": 7000, "epoch": 14, "train_loss": 0.000632595329079777, "cos_r_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_innovation_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_apical_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_Ac_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "r_norm": [0.1718350499868393, 0.168907031416893, 0.16570332646369934], "ldr": {"dL_sdil": -0.012075692415237427, "dL_bp": -0.023587502539157867, "ratio": 0.5119529884603272}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9745, "test_loss": 0.14063884963989257}], "final": {"test_acc": 0.9745, "test_loss": 0.14063884963989257, "wall_s": 20.934280395507812, "cos_r_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_innovation_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_apical_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_Ac_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_raw_s1.json b/results/nuis_ctrl_v1_rho0_raw_s1.json new file mode 100644 index 0000000..15e7c8c --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_raw_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_raw_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.247864246368408, "cos_r_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_innovation_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_apical_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_Ac_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "r_norm": [4.729090213775635, 4.788191795349121, 4.781856536865234], "ldr": {"dL_sdil": -0.035402774810791016, "dL_bp": -0.2081589698791504, "ratio": 0.17007566299614468}}, {"step": 100, "epoch": 0, "train_loss": 0.3791123926639557, "cos_r_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_innovation_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_apical_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_Ac_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "r_norm": [0.8098099231719971, 0.8149547576904297, 0.8344215154647827]}, {"step": 200, "epoch": 0, "train_loss": 0.2855910360813141, "cos_r_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_innovation_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_apical_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_Ac_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "r_norm": [0.7800662517547607, 0.7833849787712097, 0.7984472513198853]}, {"step": 300, "epoch": 0, "train_loss": 0.3106156587600708, "cos_r_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_innovation_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_apical_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_Ac_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "r_norm": [0.6266008019447327, 0.6296441555023193, 0.6423605680465698]}, {"step": 400, "epoch": 0, "train_loss": 0.1368972212076187, "cos_r_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_innovation_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_apical_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_Ac_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "r_norm": [0.4948488473892212, 0.49738022685050964, 0.5028847455978394]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9471, "test_loss": 0.18460173034667968}, {"step": 500, "epoch": 1, "train_loss": 0.22857800126075745, "cos_r_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_innovation_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_apical_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_Ac_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "r_norm": [0.5006556510925293, 0.501956582069397, 0.5072718858718872], "ldr": {"dL_sdil": -0.010978296399116516, "dL_bp": -0.00958406925201416, "ratio": 1.1454734007487841}}, {"step": 600, "epoch": 1, "train_loss": 0.26168546080589294, "cos_r_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_innovation_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_apical_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_Ac_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "r_norm": [0.4931199550628662, 0.49525341391563416, 0.5031353235244751]}, {"step": 700, "epoch": 1, "train_loss": 0.08662884682416916, "cos_r_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_innovation_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_apical_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_Ac_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "r_norm": [0.41211724281311035, 0.4123530983924866, 0.42171144485473633]}, {"step": 800, "epoch": 1, "train_loss": 0.06552650779485703, "cos_r_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_innovation_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_apical_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_Ac_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "r_norm": [0.37752482295036316, 0.37500566244125366, 0.38382813334465027]}, {"step": 900, "epoch": 1, "train_loss": 0.16771306097507477, "cos_r_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_innovation_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_apical_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_Ac_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "r_norm": [0.3334032893180847, 0.33176112174987793, 0.3353516459465027]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9596, "test_loss": 0.1381149112701416}, {"step": 1000, "epoch": 2, "train_loss": 0.0976904109120369, "cos_r_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_innovation_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_apical_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_Ac_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "r_norm": [0.3129379451274872, 0.31030431389808655, 0.3134283423423767], "ldr": {"dL_sdil": -0.005524396896362305, "dL_bp": -0.00501541793346405, "ratio": 1.101482861378755}}, {"step": 1100, "epoch": 2, "train_loss": 0.11463465541601181, "cos_r_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_innovation_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_apical_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_Ac_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "r_norm": [0.27226677536964417, 0.27177441120147705, 0.2737849950790405]}, {"step": 1200, "epoch": 2, "train_loss": 0.022953230887651443, "cos_r_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_innovation_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_apical_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_Ac_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "r_norm": [0.3118118643760681, 0.31061288714408875, 0.3118583559989929]}, {"step": 1300, "epoch": 2, "train_loss": 0.19519557058811188, "cos_r_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_innovation_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_apical_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_Ac_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "r_norm": [0.27181971073150635, 0.26876047253608704, 0.2751275300979614]}, {"step": 1400, "epoch": 2, "train_loss": 0.20355364680290222, "cos_r_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_innovation_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_apical_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_Ac_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "r_norm": [0.26793360710144043, 0.2665303349494934, 0.2706131637096405]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9619, "test_loss": 0.12944196891784668}, {"step": 1500, "epoch": 3, "train_loss": 0.04727518558502197, "cos_r_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_innovation_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_apical_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_Ac_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "r_norm": [0.2907962203025818, 0.2895806133747101, 0.2904374301433563], "ldr": {"dL_sdil": -0.00766473263502121, "dL_bp": -0.007067129015922546, "ratio": 1.0845610173172495}}, {"step": 1600, "epoch": 3, "train_loss": 0.028240162879228592, "cos_r_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_innovation_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_apical_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_Ac_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "r_norm": [0.24258095026016235, 0.2405192106962204, 0.2429429292678833]}, {"step": 1700, "epoch": 3, "train_loss": 0.08582475781440735, "cos_r_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_innovation_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_apical_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_Ac_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "r_norm": [0.23459598422050476, 0.23319926857948303, 0.23500296473503113]}, {"step": 1800, "epoch": 3, "train_loss": 0.054279766976833344, "cos_r_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_innovation_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_apical_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_Ac_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "r_norm": [0.2751310467720032, 0.2752900719642639, 0.2777182459831238]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9686, "test_loss": 0.1130959093093872}, {"step": 1900, "epoch": 4, "train_loss": 0.08464933186769485, "cos_r_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_innovation_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_apical_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_Ac_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "r_norm": [0.21713416278362274, 0.21652851998806, 0.21772976219654083]}, {"step": 2000, "epoch": 4, "train_loss": 0.08816886693239212, "cos_r_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_innovation_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_apical_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_Ac_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "r_norm": [0.20291899144649506, 0.2031886875629425, 0.20419245958328247], "ldr": {"dL_sdil": -0.004766151309013367, "dL_bp": -0.005502060055732727, "ratio": 0.8662485070564434}}, {"step": 2100, "epoch": 4, "train_loss": 0.07242057472467422, "cos_r_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_innovation_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_apical_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_Ac_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "r_norm": [0.23937690258026123, 0.23910373449325562, 0.2406611144542694]}, {"step": 2200, "epoch": 4, "train_loss": 0.07056484371423721, "cos_r_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_innovation_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_apical_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_Ac_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "r_norm": [0.23531854152679443, 0.23204778134822845, 0.23232631385326385]}, {"step": 2300, "epoch": 4, "train_loss": 0.12437652051448822, "cos_r_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_innovation_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_apical_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_Ac_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "r_norm": [0.21436047554016113, 0.21310736238956451, 0.21543720364570618]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.967, "test_loss": 0.11917487754821778}, {"step": 2400, "epoch": 5, "train_loss": 0.02945767343044281, "cos_r_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_innovation_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_apical_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_Ac_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "r_norm": [0.17013323307037354, 0.16899538040161133, 0.17079725861549377]}, {"step": 2500, "epoch": 5, "train_loss": 0.05113217979669571, "cos_r_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_innovation_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_apical_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_Ac_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "r_norm": [0.19201859831809998, 0.190620556473732, 0.19072513282299042], "ldr": {"dL_sdil": -0.0062797218561172485, "dL_bp": -0.007466904819011688, "ratio": 0.8410073528898184}}, {"step": 2600, "epoch": 5, "train_loss": 0.00991781149059534, "cos_r_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_innovation_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_apical_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_Ac_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "r_norm": [0.19189369678497314, 0.1880035698413849, 0.18973243236541748]}, {"step": 2700, "epoch": 5, "train_loss": 0.05675402656197548, "cos_r_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_innovation_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_apical_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_Ac_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "r_norm": [0.2130669355392456, 0.20993337035179138, 0.21378093957901]}, {"step": 2800, "epoch": 5, "train_loss": 0.009208951145410538, "cos_r_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_innovation_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_apical_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_Ac_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "r_norm": [0.21457375586032867, 0.21203386783599854, 0.21387223899364471]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9724, "test_loss": 0.10937111053466797}, {"step": 2900, "epoch": 6, "train_loss": 0.014543101191520691, "cos_r_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_innovation_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_apical_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_Ac_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "r_norm": [0.18077343702316284, 0.1798022985458374, 0.18212389945983887]}, {"step": 3000, "epoch": 6, "train_loss": 0.006073781289160252, "cos_r_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_innovation_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_apical_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_Ac_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "r_norm": [0.15268780291080475, 0.15006791055202484, 0.1529131680727005], "ldr": {"dL_sdil": -0.005284003913402557, "dL_bp": -0.006912320852279663, "ratio": 0.7644326741082784}}, {"step": 3100, "epoch": 6, "train_loss": 0.07234150171279907, "cos_r_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_innovation_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_apical_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_Ac_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "r_norm": [0.13482704758644104, 0.13283531367778778, 0.1336984634399414]}, {"step": 3200, "epoch": 6, "train_loss": 0.06304893642663956, "cos_r_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_innovation_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_apical_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_Ac_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "r_norm": [0.18191789090633392, 0.1812913715839386, 0.18045282363891602]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9716, "test_loss": 0.11546330757141113}, {"step": 3300, "epoch": 7, "train_loss": 0.008135353215038776, "cos_r_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_innovation_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_apical_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_Ac_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "r_norm": [0.1324104368686676, 0.13133256137371063, 0.1319148689508438]}, {"step": 3400, "epoch": 7, "train_loss": 0.003062707372009754, "cos_r_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_innovation_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_apical_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_Ac_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "r_norm": [0.17751097679138184, 0.17610973119735718, 0.17820629477500916]}, {"step": 3500, "epoch": 7, "train_loss": 0.0075003234669566154, "cos_r_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_innovation_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_apical_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_Ac_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "r_norm": [0.15693698823451996, 0.1564456969499588, 0.15661953389644623], "ldr": {"dL_sdil": -0.005416072905063629, "dL_bp": -0.0077264681458473206, "ratio": 0.7009765397110399}}, {"step": 3600, "epoch": 7, "train_loss": 0.004378090146929026, "cos_r_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_innovation_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_apical_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_Ac_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "r_norm": [0.15218180418014526, 0.15111035108566284, 0.1530647873878479]}, {"step": 3700, "epoch": 7, "train_loss": 0.005401092581450939, "cos_r_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_innovation_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_apical_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_Ac_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "r_norm": [0.15374818444252014, 0.1511906236410141, 0.15268918871879578]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9725, "test_loss": 0.12304098224639892}, {"step": 3800, "epoch": 8, "train_loss": 0.020612072199583054, "cos_r_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_innovation_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_apical_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_Ac_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "r_norm": [0.18328630924224854, 0.18096177279949188, 0.1839454621076584]}, {"step": 3900, "epoch": 8, "train_loss": 0.00907292403280735, "cos_r_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_innovation_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_apical_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_Ac_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "r_norm": [0.15745952725410461, 0.15553179383277893, 0.15759898722171783]}, {"step": 4000, "epoch": 8, "train_loss": 0.0011170797515660524, "cos_r_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_innovation_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_apical_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_Ac_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "r_norm": [0.13805150985717773, 0.13616320490837097, 0.1368236541748047], "ldr": {"dL_sdil": -0.005435429513454437, "dL_bp": -0.00862833857536316, "ratio": 0.6299508840525146}}, {"step": 4100, "epoch": 8, "train_loss": 0.06657039374113083, "cos_r_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_innovation_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_apical_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_Ac_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "r_norm": [0.14812925457954407, 0.14660516381263733, 0.14825358986854553]}, {"step": 4200, "epoch": 8, "train_loss": 0.01757517084479332, "cos_r_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_innovation_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_apical_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_Ac_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "r_norm": [0.14519406855106354, 0.14317519962787628, 0.1447223722934723]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9733, "test_loss": 0.12583541717529298}, {"step": 4300, "epoch": 9, "train_loss": 0.0043710931204259396, "cos_r_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_innovation_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_apical_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_Ac_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "r_norm": [0.17884290218353271, 0.1771182417869568, 0.17936065793037415]}, {"step": 4400, "epoch": 9, "train_loss": 0.012221035547554493, "cos_r_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_innovation_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_apical_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_Ac_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "r_norm": [0.159407377243042, 0.15782558917999268, 0.15985700488090515]}, {"step": 4500, "epoch": 9, "train_loss": 0.004442101344466209, "cos_r_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_innovation_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_apical_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_Ac_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "r_norm": [0.1378084421157837, 0.13638418912887573, 0.13881981372833252], "ldr": {"dL_sdil": -0.006090611219406128, "dL_bp": -0.009796351194381714, "ratio": 0.6217224248656115}}, {"step": 4600, "epoch": 9, "train_loss": 0.05368257686495781, "cos_r_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_innovation_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_apical_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_Ac_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "r_norm": [0.14086458086967468, 0.1384350061416626, 0.1409018635749817]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9722, "test_loss": 0.1313097234725952}, {"step": 4700, "epoch": 10, "train_loss": 0.0048868609592318535, "cos_r_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_innovation_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_apical_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_Ac_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "r_norm": [0.13858553767204285, 0.13714627921581268, 0.13974396884441376]}, {"step": 4800, "epoch": 10, "train_loss": 0.001804178231395781, "cos_r_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_innovation_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_apical_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_Ac_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "r_norm": [0.1522747427225113, 0.1504199206829071, 0.15280333161354065]}, {"step": 4900, "epoch": 10, "train_loss": 0.0028787143528461456, "cos_r_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_innovation_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_apical_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_Ac_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "r_norm": [0.1442190408706665, 0.14268440008163452, 0.14490017294883728]}, {"step": 5000, "epoch": 10, "train_loss": 0.0013339570723474026, "cos_r_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_innovation_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_apical_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_Ac_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "r_norm": [0.1472242772579193, 0.14609915018081665, 0.14829722046852112], "ldr": {"dL_sdil": -0.006713069975376129, "dL_bp": -0.011234067380428314, "ratio": 0.5975636203741715}}, {"step": 5100, "epoch": 10, "train_loss": 0.0018756217323243618, "cos_r_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_innovation_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_apical_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_Ac_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "r_norm": [0.14868545532226562, 0.14725178480148315, 0.14963632822036743]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.973, "test_loss": 0.13351503677368165}, {"step": 5200, "epoch": 11, "train_loss": 0.0014711369294673204, "cos_r_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_innovation_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_apical_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_Ac_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "r_norm": [0.12346100807189941, 0.12253932654857635, 0.12461203336715698]}, {"step": 5300, "epoch": 11, "train_loss": 0.001629142090678215, "cos_r_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_innovation_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_apical_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_Ac_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "r_norm": [0.12968139350414276, 0.1281958818435669, 0.13058608770370483]}, {"step": 5400, "epoch": 11, "train_loss": 0.0024097473360598087, "cos_r_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_innovation_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_apical_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_Ac_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "r_norm": [0.12974487245082855, 0.12847599387168884, 0.13078004121780396]}, {"step": 5500, "epoch": 11, "train_loss": 0.0010489317355677485, "cos_r_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_innovation_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_apical_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_Ac_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "r_norm": [0.13474895060062408, 0.13274598121643066, 0.13516360521316528], "ldr": {"dL_sdil": -0.006466925144195557, "dL_bp": -0.01084946095943451, "ratio": 0.5960595801372074}}, {"step": 5600, "epoch": 11, "train_loss": 0.0009077594731934369, "cos_r_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_innovation_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_apical_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_Ac_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "r_norm": [0.13371193408966064, 0.13146626949310303, 0.13360051810741425]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9739, "test_loss": 0.1354274552345276}, {"step": 5700, "epoch": 12, "train_loss": 0.0005193198448978364, "cos_r_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_innovation_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_apical_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_Ac_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "r_norm": [0.13641861081123352, 0.13459017872810364, 0.1368558406829834]}, {"step": 5800, "epoch": 12, "train_loss": 0.0007246336317621171, "cos_r_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_innovation_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_apical_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_Ac_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "r_norm": [0.1397680640220642, 0.137950599193573, 0.14015179872512817]}, {"step": 5900, "epoch": 12, "train_loss": 0.0012115520657971501, "cos_r_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_innovation_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_apical_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_Ac_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "r_norm": [0.12886938452720642, 0.12738792598247528, 0.1296568214893341]}, {"step": 6000, "epoch": 12, "train_loss": 0.0011408575810492039, "cos_r_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_innovation_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_apical_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_Ac_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "r_norm": [0.13654574751853943, 0.13504381477832794, 0.1375402808189392], "ldr": {"dL_sdil": -0.006541714072227478, "dL_bp": -0.010865777730941772, "ratio": 0.602047477337868}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9739, "test_loss": 0.13652948637008666}, {"step": 6100, "epoch": 13, "train_loss": 0.0016530295833945274, "cos_r_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_innovation_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_apical_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_Ac_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "r_norm": [0.13538393378257751, 0.13388687372207642, 0.1362152248620987]}, {"step": 6200, "epoch": 13, "train_loss": 0.0004883500514551997, "cos_r_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_innovation_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_apical_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_Ac_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "r_norm": [0.13575664162635803, 0.13420100510120392, 0.13645632565021515]}, {"step": 6300, "epoch": 13, "train_loss": 0.0005272991256788373, "cos_r_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_innovation_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_apical_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_Ac_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "r_norm": [0.1379033327102661, 0.1362687349319458, 0.13877828419208527]}, {"step": 6400, "epoch": 13, "train_loss": 0.0008385080145671964, "cos_r_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_innovation_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_apical_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_Ac_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "r_norm": [0.1363997906446457, 0.1347801685333252, 0.13696125149726868]}, {"step": 6500, "epoch": 13, "train_loss": 0.0004538414941634983, "cos_r_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_innovation_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_apical_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_Ac_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "r_norm": [0.13410302996635437, 0.13246551156044006, 0.1348181515932083], "ldr": {"dL_sdil": -0.006781101226806641, "dL_bp": -0.01180441677570343, "ratio": 0.5744545754063781}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9737, "test_loss": 0.13901660509109498}, {"step": 6600, "epoch": 14, "train_loss": 0.0003573036228772253, "cos_r_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_innovation_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_apical_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_Ac_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "r_norm": [0.13315507769584656, 0.13159938156604767, 0.1339610517024994]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008824412361718714, "cos_r_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_innovation_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_apical_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_Ac_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "r_norm": [0.1327393501996994, 0.1310739368200302, 0.13362014293670654]}, {"step": 6800, "epoch": 14, "train_loss": 0.0010087769478559494, "cos_r_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_innovation_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_apical_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_Ac_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "r_norm": [0.13503387570381165, 0.1333080381155014, 0.13559962809085846]}, {"step": 6900, "epoch": 14, "train_loss": 0.0009153951541520655, "cos_r_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_innovation_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_apical_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_Ac_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "r_norm": [0.13242390751838684, 0.1308332085609436, 0.13339316844940186]}, {"step": 7000, "epoch": 14, "train_loss": 0.00029737711884081364, "cos_r_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_innovation_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_apical_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_Ac_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "r_norm": [0.1377774327993393, 0.1363024264574051, 0.13875912129878998], "ldr": {"dL_sdil": -0.007288441061973572, "dL_bp": -0.01291123777627945, "ratio": 0.564503666361401}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9742, "test_loss": 0.14128447875976563}], "final": {"test_acc": 0.9742, "test_loss": 0.14128447875976563, "wall_s": 21.467409133911133, "cos_r_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_innovation_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_apical_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_Ac_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_raw_s2.json b/results/nuis_ctrl_v1_rho0_raw_s2.json new file mode 100644 index 0000000..dbd696c --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_raw_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_raw_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4856414794921875, "cos_r_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_innovation_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_apical_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_Ac_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "r_norm": [4.7544169425964355, 4.779862403869629, 4.861393451690674], "ldr": {"dL_sdil": -0.044302940368652344, "dL_bp": -0.24401354789733887, "ratio": 0.18155934680844618}}, {"step": 100, "epoch": 0, "train_loss": 0.3662866950035095, "cos_r_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_innovation_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_apical_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_Ac_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "r_norm": [0.7926816940307617, 0.8032606840133667, 0.8174332976341248]}, {"step": 200, "epoch": 0, "train_loss": 0.2967141568660736, "cos_r_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_innovation_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_apical_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_Ac_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "r_norm": [0.724601149559021, 0.7336956262588501, 0.7452678680419922]}, {"step": 300, "epoch": 0, "train_loss": 0.2878264784812927, "cos_r_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_innovation_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_apical_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_Ac_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "r_norm": [0.6228486895561218, 0.6298609972000122, 0.6399674415588379]}, {"step": 400, "epoch": 0, "train_loss": 0.13541436195373535, "cos_r_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_innovation_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_apical_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_Ac_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "r_norm": [0.5261503458023071, 0.5319482684135437, 0.5407342910766602]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9436, "test_loss": 0.18730589828491212}, {"step": 500, "epoch": 1, "train_loss": 0.2773134708404541, "cos_r_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_innovation_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_apical_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_Ac_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "r_norm": [0.5363684892654419, 0.5415335893630981, 0.5485605597496033], "ldr": {"dL_sdil": -0.016582071781158447, "dL_bp": -0.013470649719238281, "ratio": 1.2309778761061947}}, {"step": 600, "epoch": 1, "train_loss": 0.24808740615844727, "cos_r_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_innovation_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_apical_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_Ac_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "r_norm": [0.4696118235588074, 0.4752689301967621, 0.48475584387779236]}, {"step": 700, "epoch": 1, "train_loss": 0.059591565281152725, "cos_r_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_innovation_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_apical_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_Ac_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "r_norm": [0.4132225215435028, 0.417829692363739, 0.42398762702941895]}, {"step": 800, "epoch": 1, "train_loss": 0.08955289423465729, "cos_r_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_innovation_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_apical_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_Ac_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "r_norm": [0.38012179732322693, 0.38466107845306396, 0.3904116451740265]}, {"step": 900, "epoch": 1, "train_loss": 0.12817199528217316, "cos_r_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_innovation_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_apical_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_Ac_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "r_norm": [0.42234253883361816, 0.42529693245887756, 0.43161889910697937]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9563, "test_loss": 0.14741420364379884}, {"step": 1000, "epoch": 2, "train_loss": 0.15121077001094818, "cos_r_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_innovation_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_apical_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_Ac_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "r_norm": [0.38415396213531494, 0.38890281319618225, 0.3945630192756653], "ldr": {"dL_sdil": -0.01034557819366455, "dL_bp": -0.009327605366706848, "ratio": 1.109135494796035}}, {"step": 1100, "epoch": 2, "train_loss": 0.05521024763584137, "cos_r_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_innovation_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_apical_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_Ac_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "r_norm": [0.2886888384819031, 0.2918413281440735, 0.29741814732551575]}, {"step": 1200, "epoch": 2, "train_loss": 0.06215231865644455, "cos_r_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_innovation_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_apical_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_Ac_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "r_norm": [0.32788410782814026, 0.33229193091392517, 0.3380418121814728]}, {"step": 1300, "epoch": 2, "train_loss": 0.18930861353874207, "cos_r_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_innovation_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_apical_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_Ac_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "r_norm": [0.33047181367874146, 0.334816038608551, 0.3400202989578247]}, {"step": 1400, "epoch": 2, "train_loss": 0.20265927910804749, "cos_r_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_innovation_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_apical_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_Ac_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "r_norm": [0.3220111131668091, 0.32452118396759033, 0.3309333324432373]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9623, "test_loss": 0.1362317768096924}, {"step": 1500, "epoch": 3, "train_loss": 0.054935283958911896, "cos_r_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_innovation_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_apical_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_Ac_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "r_norm": [0.29881566762924194, 0.30197426676750183, 0.3079264760017395], "ldr": {"dL_sdil": -0.008590251207351685, "dL_bp": -0.008395388722419739, "ratio": 1.0232106566324402}}, {"step": 1600, "epoch": 3, "train_loss": 0.044838763773441315, "cos_r_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_innovation_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_apical_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_Ac_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "r_norm": [0.26855069398880005, 0.2721232771873474, 0.27481386065483093]}, {"step": 1700, "epoch": 3, "train_loss": 0.08930075168609619, "cos_r_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_innovation_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_apical_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_Ac_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "r_norm": [0.2840762436389923, 0.2851904630661011, 0.29301607608795166]}, {"step": 1800, "epoch": 3, "train_loss": 0.08093765377998352, "cos_r_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_innovation_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_apical_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_Ac_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "r_norm": [0.3048495054244995, 0.3076798915863037, 0.31106480956077576]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.968, "test_loss": 0.11007094802856446}, {"step": 1900, "epoch": 4, "train_loss": 0.04483484849333763, "cos_r_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_innovation_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_apical_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_Ac_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "r_norm": [0.22422946989536285, 0.22653135657310486, 0.23203539848327637]}, {"step": 2000, "epoch": 4, "train_loss": 0.06511980295181274, "cos_r_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_innovation_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_apical_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_Ac_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "r_norm": [0.22974814474582672, 0.2313867211341858, 0.23618879914283752], "ldr": {"dL_sdil": -0.005495421588420868, "dL_bp": -0.0060300081968307495, "ratio": 0.9113456249212315}}, {"step": 2100, "epoch": 4, "train_loss": 0.033010225743055344, "cos_r_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_innovation_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_apical_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_Ac_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "r_norm": [0.25794345140457153, 0.260486364364624, 0.26540467143058777]}, {"step": 2200, "epoch": 4, "train_loss": 0.10495644807815552, "cos_r_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_innovation_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_apical_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_Ac_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "r_norm": [0.24134549498558044, 0.2427096962928772, 0.2470351755619049]}, {"step": 2300, "epoch": 4, "train_loss": 0.13655468821525574, "cos_r_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_innovation_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_apical_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_Ac_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "r_norm": [0.2497454583644867, 0.2505592703819275, 0.25532519817352295]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.971, "test_loss": 0.1055252311706543}, {"step": 2400, "epoch": 5, "train_loss": 0.03528325632214546, "cos_r_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_innovation_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_apical_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_Ac_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "r_norm": [0.2337566614151001, 0.23517361283302307, 0.24056139588356018]}, {"step": 2500, "epoch": 5, "train_loss": 0.051970984786748886, "cos_r_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_innovation_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_apical_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_Ac_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "r_norm": [0.20650900900363922, 0.20780494809150696, 0.21189066767692566], "ldr": {"dL_sdil": -0.0069409385323524475, "dL_bp": -0.00879254937171936, "ratio": 0.7894113798982473}}, {"step": 2600, "epoch": 5, "train_loss": 0.018195580691099167, "cos_r_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_innovation_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_apical_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_Ac_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "r_norm": [0.19724297523498535, 0.1984039545059204, 0.2021041214466095]}, {"step": 2700, "epoch": 5, "train_loss": 0.0936257466673851, "cos_r_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_innovation_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_apical_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_Ac_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "r_norm": [0.20749273896217346, 0.209029883146286, 0.21408987045288086]}, {"step": 2800, "epoch": 5, "train_loss": 0.02398904599249363, "cos_r_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_innovation_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_apical_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_Ac_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "r_norm": [0.2375624030828476, 0.2401655614376068, 0.2447095513343811]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9716, "test_loss": 0.1127711856842041}, {"step": 2900, "epoch": 6, "train_loss": 0.02472660131752491, "cos_r_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_innovation_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_apical_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_Ac_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "r_norm": [0.200751394033432, 0.20256194472312927, 0.20603583753108978]}, {"step": 3000, "epoch": 6, "train_loss": 0.01555752381682396, "cos_r_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_innovation_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_apical_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_Ac_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "r_norm": [0.22714081406593323, 0.22868841886520386, 0.2330799400806427], "ldr": {"dL_sdil": -0.008373714983463287, "dL_bp": -0.011146388947963715, "ratio": 0.7512491285344071}}, {"step": 3100, "epoch": 6, "train_loss": 0.012355584651231766, "cos_r_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_innovation_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_apical_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_Ac_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "r_norm": [0.19545826315879822, 0.1969728022813797, 0.19908244907855988]}, {"step": 3200, "epoch": 6, "train_loss": 0.03212413936853409, "cos_r_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_innovation_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_apical_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_Ac_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "r_norm": [0.20170357823371887, 0.20397330820560455, 0.20707929134368896]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9719, "test_loss": 0.11250891075134277}, {"step": 3300, "epoch": 7, "train_loss": 0.014137478545308113, "cos_r_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_innovation_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_apical_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_Ac_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "r_norm": [0.19205130636692047, 0.19374202191829681, 0.1985575556755066]}, {"step": 3400, "epoch": 7, "train_loss": 0.015627946704626083, "cos_r_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_innovation_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_apical_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_Ac_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "r_norm": [0.17368774116039276, 0.17372958362102509, 0.17921704053878784]}, {"step": 3500, "epoch": 7, "train_loss": 0.009410088881850243, "cos_r_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_innovation_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_apical_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_Ac_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "r_norm": [0.15606719255447388, 0.1569913625717163, 0.16070200502872467], "ldr": {"dL_sdil": -0.005623310804367065, "dL_bp": -0.008359216153621674, "ratio": 0.6727079071798779}}, {"step": 3600, "epoch": 7, "train_loss": 0.007804212160408497, "cos_r_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_innovation_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_apical_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_Ac_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "r_norm": [0.1693708598613739, 0.16993819177150726, 0.17434334754943848]}, {"step": 3700, "epoch": 7, "train_loss": 0.002727900166064501, "cos_r_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_innovation_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_apical_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_Ac_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "r_norm": [0.19530899822711945, 0.1965683102607727, 0.20004117488861084]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9735, "test_loss": 0.12146724281311035}, {"step": 3800, "epoch": 8, "train_loss": 0.0279134102165699, "cos_r_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_innovation_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_apical_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_Ac_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "r_norm": [0.199128657579422, 0.1991393268108368, 0.20451344549655914]}, {"step": 3900, "epoch": 8, "train_loss": 0.007016683928668499, "cos_r_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_innovation_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_apical_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_Ac_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "r_norm": [0.17034371197223663, 0.17111225426197052, 0.17593686282634735]}, {"step": 4000, "epoch": 8, "train_loss": 0.0011241799220442772, "cos_r_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_innovation_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_apical_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_Ac_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "r_norm": [0.2055906504392624, 0.206048384308815, 0.2106199562549591], "ldr": {"dL_sdil": -0.009515531361103058, "dL_bp": -0.01432783156633377, "ratio": 0.6641292031560299}}, {"step": 4100, "epoch": 8, "train_loss": 0.052838996052742004, "cos_r_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_innovation_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_apical_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_Ac_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "r_norm": [0.18898266553878784, 0.1900315284729004, 0.19334247708320618]}, {"step": 4200, "epoch": 8, "train_loss": 0.010721303522586823, "cos_r_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_innovation_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_apical_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_Ac_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "r_norm": [0.1758223921060562, 0.17690947651863098, 0.1805993616580963]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9726, "test_loss": 0.12696008186340332}, {"step": 4300, "epoch": 9, "train_loss": 0.0010635580401867628, "cos_r_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_innovation_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_apical_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_Ac_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "r_norm": [0.17538368701934814, 0.17650076746940613, 0.1804800033569336]}, {"step": 4400, "epoch": 9, "train_loss": 0.0015249300049617887, "cos_r_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_innovation_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_apical_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_Ac_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "r_norm": [0.17721575498580933, 0.17846234142780304, 0.1818528175354004]}, {"step": 4500, "epoch": 9, "train_loss": 0.005601717159152031, "cos_r_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_innovation_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_apical_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_Ac_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "r_norm": [0.16339799761772156, 0.16483958065509796, 0.16933569312095642], "ldr": {"dL_sdil": -0.0075685083866119385, "dL_bp": -0.012692451477050781, "ratio": 0.5962999661882936}}, {"step": 4600, "epoch": 9, "train_loss": 0.04552677646279335, "cos_r_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_innovation_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_apical_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_Ac_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "r_norm": [0.16808894276618958, 0.16934075951576233, 0.1726376712322235]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9747, "test_loss": 0.12921665706634522}, {"step": 4700, "epoch": 10, "train_loss": 0.0016382120084017515, "cos_r_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_innovation_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_apical_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_Ac_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "r_norm": [0.17644554376602173, 0.17762485146522522, 0.18118296563625336]}, {"step": 4800, "epoch": 10, "train_loss": 0.0031079151667654514, "cos_r_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_innovation_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_apical_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_Ac_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "r_norm": [0.17002223432064056, 0.17095139622688293, 0.1749536097049713]}, {"step": 4900, "epoch": 10, "train_loss": 0.001899699796922505, "cos_r_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_innovation_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_apical_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_Ac_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "r_norm": [0.1906973421573639, 0.19160214066505432, 0.19531694054603577]}, {"step": 5000, "epoch": 10, "train_loss": 0.002346828579902649, "cos_r_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_innovation_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_apical_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_Ac_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "r_norm": [0.17863205075263977, 0.18014976382255554, 0.18399974703788757], "ldr": {"dL_sdil": -0.008974842727184296, "dL_bp": -0.015554085373878479, "ratio": 0.5770087093810505}}, {"step": 5100, "epoch": 10, "train_loss": 0.0018325410783290863, "cos_r_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_innovation_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_apical_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_Ac_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "r_norm": [0.17877718806266785, 0.1797236204147339, 0.183450847864151]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9742, "test_loss": 0.1357520221710205}, {"step": 5200, "epoch": 11, "train_loss": 0.0008780563366599381, "cos_r_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_innovation_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_apical_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_Ac_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "r_norm": [0.17967163026332855, 0.18078157305717468, 0.1842673122882843]}, {"step": 5300, "epoch": 11, "train_loss": 0.0013086458202451468, "cos_r_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_innovation_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_apical_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_Ac_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "r_norm": [0.17678910493850708, 0.1777363121509552, 0.18114155530929565]}, {"step": 5400, "epoch": 11, "train_loss": 0.0006866963813081384, "cos_r_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_innovation_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_apical_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_Ac_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "r_norm": [0.18510422110557556, 0.18618690967559814, 0.1894882768392563]}, {"step": 5500, "epoch": 11, "train_loss": 0.0008870868477970362, "cos_r_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_innovation_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_apical_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_Ac_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "r_norm": [0.17689445614814758, 0.17830178141593933, 0.18123194575309753], "ldr": {"dL_sdil": -0.008988067507743835, "dL_bp": -0.01625603437423706, "ratio": 0.5529065269441318}}, {"step": 5600, "epoch": 11, "train_loss": 0.0004585091082844883, "cos_r_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_innovation_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_apical_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_Ac_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "r_norm": [0.17601367831230164, 0.17745786905288696, 0.18061557412147522]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9754, "test_loss": 0.13874979991912842}, {"step": 5700, "epoch": 12, "train_loss": 0.0007273177616298199, "cos_r_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_innovation_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_apical_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_Ac_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "r_norm": [0.1876221001148224, 0.18882854282855988, 0.1921488344669342]}, {"step": 5800, "epoch": 12, "train_loss": 0.00034042136394418776, "cos_r_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_innovation_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_apical_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_Ac_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "r_norm": [0.1746070384979248, 0.17565712332725525, 0.17919966578483582]}, {"step": 5900, "epoch": 12, "train_loss": 0.0014644403709098697, "cos_r_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_innovation_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_apical_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_Ac_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "r_norm": [0.171538308262825, 0.17262454330921173, 0.176008403301239]}, {"step": 6000, "epoch": 12, "train_loss": 0.0009379165712743998, "cos_r_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_innovation_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_apical_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_Ac_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "r_norm": [0.17355795204639435, 0.17495247721672058, 0.17821210622787476], "ldr": {"dL_sdil": -0.008706547319889069, "dL_bp": -0.015992946922779083, "ratio": 0.5443991880876035}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9754, "test_loss": 0.14179479808807374}, {"step": 6100, "epoch": 13, "train_loss": 0.001136131351813674, "cos_r_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_innovation_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_apical_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_Ac_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "r_norm": [0.17050513625144958, 0.1718040406703949, 0.17503532767295837]}, {"step": 6200, "epoch": 13, "train_loss": 0.00029565332806669176, "cos_r_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_innovation_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_apical_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_Ac_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "r_norm": [0.1782611608505249, 0.17942413687705994, 0.18300361931324005]}, {"step": 6300, "epoch": 13, "train_loss": 0.00017874882905744016, "cos_r_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_innovation_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_apical_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_Ac_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "r_norm": [0.1737596094608307, 0.17501607537269592, 0.1786150336265564]}, {"step": 6400, "epoch": 13, "train_loss": 0.0014268780359998345, "cos_r_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_innovation_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_apical_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_Ac_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "r_norm": [0.17299015820026398, 0.17426249384880066, 0.1777229905128479]}, {"step": 6500, "epoch": 13, "train_loss": 0.00022807065397500992, "cos_r_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_innovation_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_apical_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_Ac_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "r_norm": [0.1683826446533203, 0.16965822875499725, 0.17293044924736023], "ldr": {"dL_sdil": -0.008848324418067932, "dL_bp": -0.017275981605052948, "ratio": 0.5121749154606612}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9744, "test_loss": 0.14613630256652832}, {"step": 6600, "epoch": 14, "train_loss": 0.0008223638869822025, "cos_r_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_innovation_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_apical_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_Ac_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "r_norm": [0.17072054743766785, 0.17188414931297302, 0.17549914121627808]}, {"step": 6700, "epoch": 14, "train_loss": 0.0012336993822827935, "cos_r_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_innovation_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_apical_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_Ac_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "r_norm": [0.17281609773635864, 0.17397551238536835, 0.1775282919406891]}, {"step": 6800, "epoch": 14, "train_loss": 0.0017774601001292467, "cos_r_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_innovation_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_apical_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_Ac_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "r_norm": [0.1771830916404724, 0.17863672971725464, 0.18169184029102325]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006432391237467527, "cos_r_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_innovation_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_apical_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_Ac_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "r_norm": [0.1692524403333664, 0.17056486010551453, 0.17403511703014374]}, {"step": 7000, "epoch": 14, "train_loss": 0.000249928911216557, "cos_r_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_innovation_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_apical_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_Ac_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "r_norm": [0.1676335632801056, 0.16881629824638367, 0.17234541475772858], "ldr": {"dL_sdil": -0.008975468575954437, "dL_bp": -0.017749518156051636, "ratio": 0.505673928556437}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9742, "test_loss": 0.14835414848327635}], "final": {"test_acc": 0.9742, "test_loss": 0.14835414848327635, "wall_s": 21.55682396888733, "cos_r_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_innovation_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_apical_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_Ac_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_raw_s3.json b/results/nuis_ctrl_v1_rho0_raw_s3.json new file mode 100644 index 0000000..6226882 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_raw_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_raw_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3283889293670654, "cos_r_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_innovation_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_apical_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_Ac_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "r_norm": [4.841709613800049, 4.737439155578613, 4.835896015167236], "ldr": {"dL_sdil": -0.0037169456481933594, "dL_bp": -0.22416257858276367, "ratio": 0.016581472570904674}}, {"step": 100, "epoch": 0, "train_loss": 0.33480948209762573, "cos_r_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_innovation_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_apical_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_Ac_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "r_norm": [0.7733307480812073, 0.758333683013916, 0.7694300413131714]}, {"step": 200, "epoch": 0, "train_loss": 0.32067805528640747, "cos_r_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_innovation_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_apical_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_Ac_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "r_norm": [0.7619251608848572, 0.7530097961425781, 0.7603987455368042]}, {"step": 300, "epoch": 0, "train_loss": 0.3700808882713318, "cos_r_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_innovation_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_apical_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_Ac_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "r_norm": [0.5746974945068359, 0.5715751647949219, 0.5759649276733398]}, {"step": 400, "epoch": 0, "train_loss": 0.11992092430591583, "cos_r_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_innovation_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_apical_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_Ac_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "r_norm": [0.4775196313858032, 0.47117429971694946, 0.4762042164802551]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9485, "test_loss": 0.17359396591186524}, {"step": 500, "epoch": 1, "train_loss": 0.25980252027511597, "cos_r_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_innovation_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_apical_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_Ac_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "r_norm": [0.5102050304412842, 0.5090956091880798, 0.5102525949478149], "ldr": {"dL_sdil": -0.014272913336753845, "dL_bp": -0.011847808957099915, "ratio": 1.2046880050509814}}, {"step": 600, "epoch": 1, "train_loss": 0.20059989392757416, "cos_r_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_innovation_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_apical_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_Ac_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "r_norm": [0.4354090094566345, 0.4269413948059082, 0.4325163960456848]}, {"step": 700, "epoch": 1, "train_loss": 0.08970798552036285, "cos_r_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_innovation_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_apical_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_Ac_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "r_norm": [0.4034784436225891, 0.4017947316169739, 0.4032314419746399]}, {"step": 800, "epoch": 1, "train_loss": 0.10161302983760834, "cos_r_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_innovation_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_apical_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_Ac_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "r_norm": [0.4015730917453766, 0.39593786001205444, 0.4005199074745178]}, {"step": 900, "epoch": 1, "train_loss": 0.12729158997535706, "cos_r_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_innovation_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_apical_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_Ac_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "r_norm": [0.3631213307380676, 0.3588687479496002, 0.36084190011024475]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9597, "test_loss": 0.1398345920562744}, {"step": 1000, "epoch": 2, "train_loss": 0.13055379688739777, "cos_r_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_innovation_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_apical_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_Ac_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "r_norm": [0.3458414077758789, 0.339888334274292, 0.3438340425491333], "ldr": {"dL_sdil": -0.00751737505197525, "dL_bp": -0.006968997418880463, "ratio": 1.0786881670538604}}, {"step": 1100, "epoch": 2, "train_loss": 0.08165145665407181, "cos_r_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_innovation_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_apical_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_Ac_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "r_norm": [0.27777719497680664, 0.27354109287261963, 0.2770301103591919]}, {"step": 1200, "epoch": 2, "train_loss": 0.03162624314427376, "cos_r_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_innovation_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_apical_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_Ac_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "r_norm": [0.3397935926914215, 0.3343070149421692, 0.33784517645835876]}, {"step": 1300, "epoch": 2, "train_loss": 0.12580201029777527, "cos_r_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_innovation_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_apical_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_Ac_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "r_norm": [0.27239900827407837, 0.26863694190979004, 0.2723345160484314]}, {"step": 1400, "epoch": 2, "train_loss": 0.20885255932807922, "cos_r_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_innovation_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_apical_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_Ac_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "r_norm": [0.28959929943084717, 0.28440648317337036, 0.2864704132080078]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9624, "test_loss": 0.127296443939209}, {"step": 1500, "epoch": 3, "train_loss": 0.07511237263679504, "cos_r_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_innovation_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_apical_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_Ac_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "r_norm": [0.31371599435806274, 0.30785977840423584, 0.31208980083465576], "ldr": {"dL_sdil": -0.007627889513969421, "dL_bp": -0.006990775465965271, "ratio": 1.0911363912495764}}, {"step": 1600, "epoch": 3, "train_loss": 0.05954476445913315, "cos_r_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_innovation_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_apical_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_Ac_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "r_norm": [0.2844116687774658, 0.28088948130607605, 0.2819110155105591]}, {"step": 1700, "epoch": 3, "train_loss": 0.05954836308956146, "cos_r_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_innovation_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_apical_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_Ac_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "r_norm": [0.27005547285079956, 0.2676847577095032, 0.2688477635383606]}, {"step": 1800, "epoch": 3, "train_loss": 0.04903425648808479, "cos_r_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_innovation_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_apical_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_Ac_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "r_norm": [0.2799452543258667, 0.2738724946975708, 0.27548953890800476]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9699, "test_loss": 0.10895987529754639}, {"step": 1900, "epoch": 4, "train_loss": 0.05801629275083542, "cos_r_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_innovation_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_apical_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_Ac_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "r_norm": [0.22653728723526, 0.22427527606487274, 0.2236478626728058]}, {"step": 2000, "epoch": 4, "train_loss": 0.10690974444150925, "cos_r_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_innovation_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_apical_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_Ac_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "r_norm": [0.2192738950252533, 0.21646450459957123, 0.21635594964027405], "ldr": {"dL_sdil": -0.004961423575878143, "dL_bp": -0.005419991910457611, "ratio": 0.9153931699243532}}, {"step": 2100, "epoch": 4, "train_loss": 0.04904601722955704, "cos_r_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_innovation_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_apical_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_Ac_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "r_norm": [0.25469183921813965, 0.24852119386196136, 0.2508085072040558]}, {"step": 2200, "epoch": 4, "train_loss": 0.07027833163738251, "cos_r_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_innovation_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_apical_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_Ac_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "r_norm": [0.22217589616775513, 0.21898141503334045, 0.2202150672674179]}, {"step": 2300, "epoch": 4, "train_loss": 0.1405230015516281, "cos_r_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_innovation_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_apical_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_Ac_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "r_norm": [0.2517160177230835, 0.24741223454475403, 0.2489166259765625]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9701, "test_loss": 0.10518000926971435}, {"step": 2400, "epoch": 5, "train_loss": 0.015669789165258408, "cos_r_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_innovation_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_apical_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_Ac_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "r_norm": [0.2449054718017578, 0.24151459336280823, 0.24172821640968323]}, {"step": 2500, "epoch": 5, "train_loss": 0.04790015518665314, "cos_r_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_innovation_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_apical_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_Ac_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "r_norm": [0.2145114541053772, 0.21184881031513214, 0.21148696541786194], "ldr": {"dL_sdil": -0.005982436239719391, "dL_bp": -0.0073370859026908875, "ratio": 0.8153695239584592}}, {"step": 2600, "epoch": 5, "train_loss": 0.014841018244624138, "cos_r_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_innovation_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_apical_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_Ac_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "r_norm": [0.19980891048908234, 0.19781577587127686, 0.19805651903152466]}, {"step": 2700, "epoch": 5, "train_loss": 0.07935170829296112, "cos_r_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_innovation_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_apical_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_Ac_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "r_norm": [0.23918147385120392, 0.23707768321037292, 0.2370166778564453]}, {"step": 2800, "epoch": 5, "train_loss": 0.005857329349964857, "cos_r_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_innovation_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_apical_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_Ac_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "r_norm": [0.21699143946170807, 0.2136717587709427, 0.21461525559425354]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.971, "test_loss": 0.10945739488601684}, {"step": 2900, "epoch": 6, "train_loss": 0.0123747568577528, "cos_r_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_innovation_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_apical_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_Ac_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "r_norm": [0.23866206407546997, 0.23430858552455902, 0.23588283360004425]}, {"step": 3000, "epoch": 6, "train_loss": 0.008246966637670994, "cos_r_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_innovation_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_apical_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_Ac_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "r_norm": [0.24846184253692627, 0.2451106160879135, 0.24654726684093475], "ldr": {"dL_sdil": -0.008115455508232117, "dL_bp": -0.010389789938926697, "ratio": 0.7810990940082926}}, {"step": 3100, "epoch": 6, "train_loss": 0.011437802575528622, "cos_r_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_innovation_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_apical_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_Ac_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "r_norm": [0.19606490433216095, 0.1932457983493805, 0.19307956099510193]}, {"step": 3200, "epoch": 6, "train_loss": 0.032173387706279755, "cos_r_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_innovation_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_apical_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_Ac_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "r_norm": [0.2475024312734604, 0.24428103864192963, 0.24396109580993652]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.971, "test_loss": 0.11738142681121826}, {"step": 3300, "epoch": 7, "train_loss": 0.0035855344031006098, "cos_r_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_innovation_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_apical_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_Ac_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "r_norm": [0.1960332989692688, 0.1931273192167282, 0.19294635951519012]}, {"step": 3400, "epoch": 7, "train_loss": 0.004453396890312433, "cos_r_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_innovation_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_apical_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_Ac_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "r_norm": [0.22042231261730194, 0.21685011684894562, 0.21672581136226654]}, {"step": 3500, "epoch": 7, "train_loss": 0.00869251973927021, "cos_r_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_innovation_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_apical_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_Ac_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "r_norm": [0.21137651801109314, 0.2083500772714615, 0.20858825743198395], "ldr": {"dL_sdil": -0.006606429815292358, "dL_bp": -0.009430110454559326, "ratio": 0.7005675964376686}}, {"step": 3600, "epoch": 7, "train_loss": 0.004775376059114933, "cos_r_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_innovation_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_apical_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_Ac_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "r_norm": [0.2073669135570526, 0.20420829951763153, 0.20337089896202087]}, {"step": 3700, "epoch": 7, "train_loss": 0.0054521383717656136, "cos_r_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_innovation_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_apical_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_Ac_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "r_norm": [0.20405349135398865, 0.20031914114952087, 0.20035898685455322]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9734, "test_loss": 0.11372013940811157}, {"step": 3800, "epoch": 8, "train_loss": 0.015979381278157234, "cos_r_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_innovation_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_apical_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_Ac_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "r_norm": [0.19954121112823486, 0.19605880975723267, 0.1963454782962799]}, {"step": 3900, "epoch": 8, "train_loss": 0.017817214131355286, "cos_r_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_innovation_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_apical_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_Ac_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "r_norm": [0.20743486285209656, 0.20354314148426056, 0.20332831144332886]}, {"step": 4000, "epoch": 8, "train_loss": 0.0004270627978257835, "cos_r_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_innovation_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_apical_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_Ac_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "r_norm": [0.19953644275665283, 0.19602517783641815, 0.1954791247844696], "ldr": {"dL_sdil": -0.008086413145065308, "dL_bp": -0.013367392122745514, "ratio": 0.6049357324758756}}, {"step": 4100, "epoch": 8, "train_loss": 0.07452463358640671, "cos_r_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_innovation_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_apical_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_Ac_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "r_norm": [0.20760531723499298, 0.20430520176887512, 0.20437899231910706]}, {"step": 4200, "epoch": 8, "train_loss": 0.03605803847312927, "cos_r_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_innovation_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_apical_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_Ac_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "r_norm": [0.19332198798656464, 0.1901654452085495, 0.18981432914733887]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9724, "test_loss": 0.11489360466003418}, {"step": 4300, "epoch": 9, "train_loss": 0.0038651630748063326, "cos_r_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_innovation_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_apical_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_Ac_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "r_norm": [0.21041682362556458, 0.20728406310081482, 0.20620422065258026]}, {"step": 4400, "epoch": 9, "train_loss": 0.0014019431546330452, "cos_r_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_innovation_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_apical_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_Ac_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "r_norm": [0.19830866158008575, 0.19524846971035004, 0.19488275051116943]}, {"step": 4500, "epoch": 9, "train_loss": 0.0025219277013093233, "cos_r_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_innovation_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_apical_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_Ac_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "r_norm": [0.19933810830116272, 0.195916086435318, 0.19609415531158447], "ldr": {"dL_sdil": -0.008369803428649902, "dL_bp": -0.014350831508636475, "ratio": 0.5832277679250063}}, {"step": 4600, "epoch": 9, "train_loss": 0.06718802452087402, "cos_r_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_innovation_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_apical_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_Ac_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "r_norm": [0.19856536388397217, 0.19568754732608795, 0.19562330842018127]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9747, "test_loss": 0.11986219282150269}, {"step": 4700, "epoch": 10, "train_loss": 0.0012022472219541669, "cos_r_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_innovation_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_apical_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_Ac_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "r_norm": [0.19958841800689697, 0.19681866466999054, 0.19656816124916077]}, {"step": 4800, "epoch": 10, "train_loss": 0.0013852894771844149, "cos_r_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_innovation_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_apical_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_Ac_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "r_norm": [0.19555793702602386, 0.1924840807914734, 0.1920878291130066]}, {"step": 4900, "epoch": 10, "train_loss": 0.0019364049658179283, "cos_r_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_innovation_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_apical_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_Ac_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "r_norm": [0.19706960022449493, 0.1942807137966156, 0.19380313158035278]}, {"step": 5000, "epoch": 10, "train_loss": 0.0011562240542843938, "cos_r_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_innovation_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_apical_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_Ac_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "r_norm": [0.19638454914093018, 0.19321104884147644, 0.1927313357591629], "ldr": {"dL_sdil": -0.008506037294864655, "dL_bp": -0.015625961124897003, "ratio": 0.544352902639179}}, {"step": 5100, "epoch": 10, "train_loss": 0.0011195843107998371, "cos_r_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_innovation_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_apical_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_Ac_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "r_norm": [0.21668797731399536, 0.21310555934906006, 0.21348032355308533]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9736, "test_loss": 0.12488943996429443}, {"step": 5200, "epoch": 11, "train_loss": 0.0005131271318532526, "cos_r_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_innovation_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_apical_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_Ac_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "r_norm": [0.2030562162399292, 0.1994873583316803, 0.19953997433185577]}, {"step": 5300, "epoch": 11, "train_loss": 0.0014590469654649496, "cos_r_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_innovation_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_apical_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_Ac_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "r_norm": [0.20533114671707153, 0.20210321247577667, 0.20187830924987793]}, {"step": 5400, "epoch": 11, "train_loss": 0.0009648285340517759, "cos_r_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_innovation_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_apical_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_Ac_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "r_norm": [0.2028876543045044, 0.19992581009864807, 0.19973811507225037]}, {"step": 5500, "epoch": 11, "train_loss": 0.002808267716318369, "cos_r_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_innovation_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_apical_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_Ac_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "r_norm": [0.19786036014556885, 0.19502316415309906, 0.19484545290470123], "ldr": {"dL_sdil": -0.009292647242546082, "dL_bp": -0.017724603414535522, "ratio": 0.5242795579237279}}, {"step": 5600, "epoch": 11, "train_loss": 0.0007443465292453766, "cos_r_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_innovation_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_apical_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_Ac_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "r_norm": [0.19685988128185272, 0.1940140724182129, 0.19382789731025696]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9741, "test_loss": 0.12913100547790526}, {"step": 5700, "epoch": 12, "train_loss": 0.00026087171863764524, "cos_r_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_innovation_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_apical_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_Ac_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "r_norm": [0.20257383584976196, 0.19969847798347473, 0.19937360286712646]}, {"step": 5800, "epoch": 12, "train_loss": 0.0011157530825585127, "cos_r_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_innovation_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_apical_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_Ac_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "r_norm": [0.20035849511623383, 0.1976122111082077, 0.1971856951713562]}, {"step": 5900, "epoch": 12, "train_loss": 0.0022946004755795, "cos_r_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_innovation_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_apical_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_Ac_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "r_norm": [0.20160531997680664, 0.19882169365882874, 0.1985234022140503]}, {"step": 6000, "epoch": 12, "train_loss": 0.0015348243759945035, "cos_r_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_innovation_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_apical_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_Ac_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "r_norm": [0.21117167174816132, 0.208292156457901, 0.20826399326324463], "ldr": {"dL_sdil": -0.010774381458759308, "dL_bp": -0.021835722029209137, "ratio": 0.49342913618091805}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9747, "test_loss": 0.13031967477798462}, {"step": 6100, "epoch": 13, "train_loss": 0.0005886715371161699, "cos_r_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_innovation_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_apical_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_Ac_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "r_norm": [0.20275822281837463, 0.1999770700931549, 0.1997852921485901]}, {"step": 6200, "epoch": 13, "train_loss": 0.00026136147789657116, "cos_r_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_innovation_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_apical_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_Ac_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "r_norm": [0.19926416873931885, 0.19659274816513062, 0.1963043063879013]}, {"step": 6300, "epoch": 13, "train_loss": 0.0008323256624862552, "cos_r_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_innovation_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_apical_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_Ac_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "r_norm": [0.19856512546539307, 0.19585520029067993, 0.19567559659481049]}, {"step": 6400, "epoch": 13, "train_loss": 0.0005636445712298155, "cos_r_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_innovation_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_apical_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_Ac_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "r_norm": [0.2014485001564026, 0.19853657484054565, 0.19835329055786133]}, {"step": 6500, "epoch": 13, "train_loss": 0.00021077357814647257, "cos_r_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_innovation_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_apical_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_Ac_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "r_norm": [0.19778192043304443, 0.19491931796073914, 0.19479045271873474], "ldr": {"dL_sdil": -0.010284215211868286, "dL_bp": -0.021059155464172363, "ratio": 0.4883488907883639}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9752, "test_loss": 0.13342348890304565}, {"step": 6600, "epoch": 14, "train_loss": 0.0004418176249600947, "cos_r_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_innovation_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_apical_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_Ac_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "r_norm": [0.20090188086032867, 0.19807308912277222, 0.19789955019950867]}, {"step": 6700, "epoch": 14, "train_loss": 0.001412956160493195, "cos_r_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_innovation_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_apical_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_Ac_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "r_norm": [0.2016269862651825, 0.19889488816261292, 0.19875365495681763]}, {"step": 6800, "epoch": 14, "train_loss": 0.0006913552642799914, "cos_r_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_innovation_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_apical_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_Ac_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "r_norm": [0.20099763572216034, 0.198038712143898, 0.19800257682800293]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006930493982508779, "cos_r_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_innovation_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_apical_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_Ac_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "r_norm": [0.19700448215007782, 0.19417372345924377, 0.19408582150936127]}, {"step": 7000, "epoch": 14, "train_loss": 0.00032816026941873133, "cos_r_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_innovation_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_apical_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_Ac_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "r_norm": [0.19693708419799805, 0.19410592317581177, 0.19396330416202545], "ldr": {"dL_sdil": -0.010280802845954895, "dL_bp": -0.021565861999988556, "ratio": 0.47671652753598953}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9753, "test_loss": 0.1358968108177185}], "final": {"test_acc": 0.9753, "test_loss": 0.1358968108177185, "wall_s": 21.953113555908203, "cos_r_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_innovation_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_apical_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_Ac_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_raw_s4.json b/results/nuis_ctrl_v1_rho0_raw_s4.json new file mode 100644 index 0000000..0a585d5 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_raw_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_raw_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4173901081085205, "cos_r_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_innovation_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_apical_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_Ac_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "r_norm": [4.844976425170898, 4.853462219238281, 4.948795318603516], "ldr": {"dL_sdil": -0.00283050537109375, "dL_bp": -0.25957250595092773, "ratio": 0.01090448836530036}}, {"step": 100, "epoch": 0, "train_loss": 0.4020535349845886, "cos_r_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_innovation_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_apical_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_Ac_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "r_norm": [0.7995222806930542, 0.8062721490859985, 0.8229418396949768]}, {"step": 200, "epoch": 0, "train_loss": 0.2508821189403534, "cos_r_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_innovation_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_apical_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_Ac_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "r_norm": [0.7765392065048218, 0.7898226380348206, 0.8046278357505798]}, {"step": 300, "epoch": 0, "train_loss": 0.25044649839401245, "cos_r_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_innovation_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_apical_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_Ac_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "r_norm": [0.6173285841941833, 0.623796284198761, 0.6384584903717041]}, {"step": 400, "epoch": 0, "train_loss": 0.17270351946353912, "cos_r_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_innovation_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_apical_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_Ac_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "r_norm": [0.5093363523483276, 0.5141817331314087, 0.5286765098571777]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9455, "test_loss": 0.1889075668334961}, {"step": 500, "epoch": 1, "train_loss": 0.21437089145183563, "cos_r_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_innovation_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_apical_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_Ac_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "r_norm": [0.520584225654602, 0.5245216488838196, 0.5380268096923828], "ldr": {"dL_sdil": -0.012444496154785156, "dL_bp": -0.010713472962379456, "ratio": 1.161574421150286}}, {"step": 600, "epoch": 1, "train_loss": 0.22007516026496887, "cos_r_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_innovation_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_apical_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_Ac_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "r_norm": [0.5072118639945984, 0.507872462272644, 0.5214201211929321]}, {"step": 700, "epoch": 1, "train_loss": 0.10645534098148346, "cos_r_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_innovation_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_apical_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_Ac_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "r_norm": [0.3919033408164978, 0.3957861661911011, 0.4059203267097473]}, {"step": 800, "epoch": 1, "train_loss": 0.099101722240448, "cos_r_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_innovation_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_apical_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_Ac_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "r_norm": [0.35605648159980774, 0.3594273328781128, 0.36916738748550415]}, {"step": 900, "epoch": 1, "train_loss": 0.11939629912376404, "cos_r_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_innovation_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_apical_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_Ac_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "r_norm": [0.35462111234664917, 0.356706440448761, 0.3648630678653717]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9565, "test_loss": 0.1442838306427002}, {"step": 1000, "epoch": 2, "train_loss": 0.10045582056045532, "cos_r_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_innovation_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_apical_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_Ac_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "r_norm": [0.28615742921829224, 0.288568913936615, 0.29509449005126953], "ldr": {"dL_sdil": -0.0052221789956092834, "dL_bp": -0.004839010536670685, "ratio": 1.0791832247594617}}, {"step": 1100, "epoch": 2, "train_loss": 0.09472673386335373, "cos_r_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_innovation_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_apical_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_Ac_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "r_norm": [0.29523903131484985, 0.295265257358551, 0.30285394191741943]}, {"step": 1200, "epoch": 2, "train_loss": 0.02559100277721882, "cos_r_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_innovation_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_apical_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_Ac_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "r_norm": [0.30911892652511597, 0.31053924560546875, 0.3160984516143799]}, {"step": 1300, "epoch": 2, "train_loss": 0.15685486793518066, "cos_r_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_innovation_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_apical_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_Ac_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "r_norm": [0.2874014675617218, 0.290073037147522, 0.2968340814113617]}, {"step": 1400, "epoch": 2, "train_loss": 0.23597419261932373, "cos_r_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_innovation_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_apical_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_Ac_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "r_norm": [0.3454799950122833, 0.34709277749061584, 0.35570353269577026]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9584, "test_loss": 0.1362682201385498}, {"step": 1500, "epoch": 3, "train_loss": 0.049877192825078964, "cos_r_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_innovation_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_apical_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_Ac_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "r_norm": [0.32026493549346924, 0.32111209630966187, 0.32673612236976624], "ldr": {"dL_sdil": -0.008508473634719849, "dL_bp": -0.007917135953903198, "ratio": 1.0746908584442227}}, {"step": 1600, "epoch": 3, "train_loss": 0.04681463539600372, "cos_r_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_innovation_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_apical_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_Ac_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "r_norm": [0.2679218351840973, 0.2675299048423767, 0.2748056650161743]}, {"step": 1700, "epoch": 3, "train_loss": 0.042607326060533524, "cos_r_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_innovation_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_apical_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_Ac_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "r_norm": [0.25324034690856934, 0.25399982929229736, 0.25864163041114807]}, {"step": 1800, "epoch": 3, "train_loss": 0.05712303891777992, "cos_r_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_innovation_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_apical_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_Ac_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "r_norm": [0.26621386408805847, 0.2666720151901245, 0.27278897166252136]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9686, "test_loss": 0.10741588859558106}, {"step": 1900, "epoch": 4, "train_loss": 0.0610680989921093, "cos_r_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_innovation_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_apical_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_Ac_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "r_norm": [0.255119651556015, 0.25568583607673645, 0.26110944151878357]}, {"step": 2000, "epoch": 4, "train_loss": 0.059706609696149826, "cos_r_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_innovation_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_apical_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_Ac_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "r_norm": [0.21822944283485413, 0.2185012400150299, 0.22326743602752686], "ldr": {"dL_sdil": -0.004507958889007568, "dL_bp": -0.004968710243701935, "ratio": 0.9072694256465469}}, {"step": 2100, "epoch": 4, "train_loss": 0.032644227147102356, "cos_r_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_innovation_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_apical_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_Ac_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "r_norm": [0.24712875485420227, 0.24843883514404297, 0.2534225881099701]}, {"step": 2200, "epoch": 4, "train_loss": 0.028950463980436325, "cos_r_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_innovation_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_apical_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_Ac_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "r_norm": [0.24966076016426086, 0.2504446804523468, 0.25478261709213257]}, {"step": 2300, "epoch": 4, "train_loss": 0.0650198757648468, "cos_r_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_innovation_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_apical_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_Ac_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "r_norm": [0.20470218360424042, 0.2031494677066803, 0.20760329067707062]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9693, "test_loss": 0.10958373928070068}, {"step": 2400, "epoch": 5, "train_loss": 0.03290189430117607, "cos_r_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_innovation_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_apical_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_Ac_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "r_norm": [0.21331188082695007, 0.21278373897075653, 0.2184755653142929]}, {"step": 2500, "epoch": 5, "train_loss": 0.03874827176332474, "cos_r_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_innovation_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_apical_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_Ac_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "r_norm": [0.22789107263088226, 0.2283928245306015, 0.23281589150428772], "ldr": {"dL_sdil": -0.0070598796010017395, "dL_bp": -0.008158765733242035, "ratio": 0.8653122092030416}}, {"step": 2600, "epoch": 5, "train_loss": 0.021128688007593155, "cos_r_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_innovation_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_apical_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_Ac_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "r_norm": [0.2212328165769577, 0.2204180359840393, 0.2251305729150772]}, {"step": 2700, "epoch": 5, "train_loss": 0.07236946374177933, "cos_r_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_innovation_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_apical_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_Ac_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "r_norm": [0.2149030566215515, 0.2140052616596222, 0.2162187099456787]}, {"step": 2800, "epoch": 5, "train_loss": 0.008522935211658478, "cos_r_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_innovation_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_apical_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_Ac_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "r_norm": [0.205214723944664, 0.2052396833896637, 0.2100837081670761]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9724, "test_loss": 0.10797543697357177}, {"step": 2900, "epoch": 6, "train_loss": 0.018756665289402008, "cos_r_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_innovation_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_apical_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_Ac_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "r_norm": [0.22938458621501923, 0.22977539896965027, 0.23387743532657623]}, {"step": 3000, "epoch": 6, "train_loss": 0.005393678322434425, "cos_r_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_innovation_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_apical_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_Ac_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "r_norm": [0.19885726273059845, 0.1973133385181427, 0.20168818533420563], "ldr": {"dL_sdil": -0.005929701030254364, "dL_bp": -0.008110091090202332, "ratio": 0.7311509777514015}}, {"step": 3100, "epoch": 6, "train_loss": 0.03258326277136803, "cos_r_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_innovation_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_apical_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_Ac_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "r_norm": [0.22770541906356812, 0.22627578675746918, 0.22949440777301788]}, {"step": 3200, "epoch": 6, "train_loss": 0.05724141746759415, "cos_r_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_innovation_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_apical_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_Ac_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "r_norm": [0.23419582843780518, 0.23333817720413208, 0.23697063326835632]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9735, "test_loss": 0.1085232892036438}, {"step": 3300, "epoch": 7, "train_loss": 0.044839728623628616, "cos_r_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_innovation_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_apical_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_Ac_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "r_norm": [0.18712347745895386, 0.185409277677536, 0.1905364990234375]}, {"step": 3400, "epoch": 7, "train_loss": 0.008097128942608833, "cos_r_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_innovation_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_apical_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_Ac_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "r_norm": [0.19379451870918274, 0.19184225797653198, 0.19568520784378052]}, {"step": 3500, "epoch": 7, "train_loss": 0.010430501773953438, "cos_r_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_innovation_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_apical_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_Ac_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "r_norm": [0.1839253008365631, 0.18301129341125488, 0.18623384833335876], "ldr": {"dL_sdil": -0.006262347102165222, "dL_bp": -0.009936518967151642, "ratio": 0.6302355103298675}}, {"step": 3600, "epoch": 7, "train_loss": 0.0012780515244230628, "cos_r_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_innovation_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_apical_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_Ac_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "r_norm": [0.1790696531534195, 0.1764722466468811, 0.17974436283111572]}, {"step": 3700, "epoch": 7, "train_loss": 0.018456075340509415, "cos_r_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_innovation_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_apical_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_Ac_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "r_norm": [0.18262530863285065, 0.18070702254772186, 0.1839236468076706]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.973, "test_loss": 0.11429949645996093}, {"step": 3800, "epoch": 8, "train_loss": 0.019817866384983063, "cos_r_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_innovation_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_apical_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_Ac_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "r_norm": [0.1908964067697525, 0.19014066457748413, 0.19408388435840607]}, {"step": 3900, "epoch": 8, "train_loss": 0.019774047657847404, "cos_r_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_innovation_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_apical_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_Ac_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "r_norm": [0.19253888726234436, 0.19145795702934265, 0.19445383548736572]}, {"step": 4000, "epoch": 8, "train_loss": 0.0010947450064122677, "cos_r_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_innovation_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_apical_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_Ac_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "r_norm": [0.18950730562210083, 0.18815365433692932, 0.19119828939437866], "ldr": {"dL_sdil": -0.007813781499862671, "dL_bp": -0.014065638184547424, "ratio": 0.5555227141024378}}, {"step": 4100, "epoch": 8, "train_loss": 0.09640847891569138, "cos_r_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_innovation_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_apical_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_Ac_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "r_norm": [0.1922830492258072, 0.19028913974761963, 0.19349952042102814]}, {"step": 4200, "epoch": 8, "train_loss": 0.014551833271980286, "cos_r_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_innovation_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_apical_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_Ac_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "r_norm": [0.19427847862243652, 0.19271467626094818, 0.19574645161628723]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9729, "test_loss": 0.12127476367950439}, {"step": 4300, "epoch": 9, "train_loss": 0.0006672337185591459, "cos_r_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_innovation_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_apical_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_Ac_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "r_norm": [0.19403856992721558, 0.19274917244911194, 0.1965358704328537]}, {"step": 4400, "epoch": 9, "train_loss": 0.004814686719328165, "cos_r_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_innovation_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_apical_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_Ac_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "r_norm": [0.1811094731092453, 0.1802864819765091, 0.18380987644195557]}, {"step": 4500, "epoch": 9, "train_loss": 0.007938915863633156, "cos_r_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_innovation_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_apical_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_Ac_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "r_norm": [0.18736639618873596, 0.185921311378479, 0.1901116669178009], "ldr": {"dL_sdil": -0.008901968598365784, "dL_bp": -0.017541170120239258, "ratio": 0.5074900099221181}}, {"step": 4600, "epoch": 9, "train_loss": 0.0795854851603508, "cos_r_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_innovation_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_apical_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_Ac_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "r_norm": [0.184382826089859, 0.1827830672264099, 0.18596649169921875]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9742, "test_loss": 0.12309427490234374}, {"step": 4700, "epoch": 10, "train_loss": 0.0007766007911413908, "cos_r_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_innovation_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_apical_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_Ac_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "r_norm": [0.19498443603515625, 0.19344991445541382, 0.19718916714191437]}, {"step": 4800, "epoch": 10, "train_loss": 0.002889199648052454, "cos_r_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_innovation_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_apical_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_Ac_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "r_norm": [0.19870562851428986, 0.1971784234046936, 0.20026731491088867]}, {"step": 4900, "epoch": 10, "train_loss": 0.0008581852307543159, "cos_r_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_innovation_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_apical_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_Ac_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "r_norm": [0.19465115666389465, 0.19327929615974426, 0.19643467664718628]}, {"step": 5000, "epoch": 10, "train_loss": 0.0007646966842003167, "cos_r_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_innovation_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_apical_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_Ac_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "r_norm": [0.19242608547210693, 0.19134974479675293, 0.19527661800384521], "ldr": {"dL_sdil": -0.009105280041694641, "dL_bp": -0.018391385674476624, "ratio": 0.4950839595697705}}, {"step": 5100, "epoch": 10, "train_loss": 0.0015073801623657346, "cos_r_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_innovation_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_apical_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_Ac_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "r_norm": [0.17740818858146667, 0.17602333426475525, 0.17976273596286774]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9745, "test_loss": 0.12885003728866576}, {"step": 5200, "epoch": 11, "train_loss": 0.0014642335008829832, "cos_r_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_innovation_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_apical_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_Ac_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "r_norm": [0.17490693926811218, 0.17364893853664398, 0.17717021703720093]}, {"step": 5300, "epoch": 11, "train_loss": 0.001281333388760686, "cos_r_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_innovation_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_apical_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_Ac_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "r_norm": [0.1789165437221527, 0.1773441880941391, 0.1806153953075409]}, {"step": 5400, "epoch": 11, "train_loss": 0.0045455265790224075, "cos_r_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_innovation_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_apical_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_Ac_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "r_norm": [0.18679404258728027, 0.18510667979717255, 0.18804368376731873]}, {"step": 5500, "epoch": 11, "train_loss": 0.002247859025374055, "cos_r_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_innovation_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_apical_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_Ac_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "r_norm": [0.1849762499332428, 0.18319714069366455, 0.18662986159324646], "ldr": {"dL_sdil": -0.00928555428981781, "dL_bp": -0.02012455463409424, "ratio": 0.4614042128458627}}, {"step": 5600, "epoch": 11, "train_loss": 0.0012828883482143283, "cos_r_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_innovation_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_apical_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_Ac_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "r_norm": [0.1720443069934845, 0.1705038845539093, 0.17383548617362976]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9739, "test_loss": 0.1312964542388916}, {"step": 5700, "epoch": 12, "train_loss": 0.0005914976936765015, "cos_r_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_innovation_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_apical_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_Ac_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "r_norm": [0.18014508485794067, 0.1789059042930603, 0.18226537108421326]}, {"step": 5800, "epoch": 12, "train_loss": 0.0010177994845435023, "cos_r_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_innovation_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_apical_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_Ac_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "r_norm": [0.19342085719108582, 0.19208499789237976, 0.19565963745117188]}, {"step": 5900, "epoch": 12, "train_loss": 0.0026742268819361925, "cos_r_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_innovation_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_apical_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_Ac_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "r_norm": [0.18534237146377563, 0.18388569355010986, 0.18735302984714508]}, {"step": 6000, "epoch": 12, "train_loss": 0.0006315991631709039, "cos_r_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_innovation_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_apical_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_Ac_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "r_norm": [0.18172697722911835, 0.1802336871623993, 0.18384242057800293], "ldr": {"dL_sdil": -0.009645640850067139, "dL_bp": -0.021210603415966034, "ratio": 0.4547556078865015}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9741, "test_loss": 0.13412653560638427}, {"step": 6100, "epoch": 13, "train_loss": 0.0009790371404960752, "cos_r_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_innovation_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_apical_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_Ac_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "r_norm": [0.18465885519981384, 0.18316440284252167, 0.18671941757202148]}, {"step": 6200, "epoch": 13, "train_loss": 0.0008633113466203213, "cos_r_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_innovation_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_apical_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_Ac_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "r_norm": [0.18658879399299622, 0.18528807163238525, 0.18887931108474731]}, {"step": 6300, "epoch": 13, "train_loss": 0.0002778768539428711, "cos_r_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_innovation_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_apical_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_Ac_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "r_norm": [0.18958455324172974, 0.188012033700943, 0.19160720705986023]}, {"step": 6400, "epoch": 13, "train_loss": 0.00041367538506165147, "cos_r_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_innovation_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_apical_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_Ac_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "r_norm": [0.19554319977760315, 0.1940520703792572, 0.19749662280082703]}, {"step": 6500, "epoch": 13, "train_loss": 0.00021037989063188434, "cos_r_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_innovation_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_apical_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_Ac_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "r_norm": [0.18189536035060883, 0.18059924244880676, 0.18395429849624634], "ldr": {"dL_sdil": -0.009790733456611633, "dL_bp": -0.022421933710575104, "ratio": 0.43665874598469273}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9738, "test_loss": 0.1366198040008545}, {"step": 6600, "epoch": 14, "train_loss": 0.0006715196650475264, "cos_r_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_innovation_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_apical_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_Ac_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "r_norm": [0.17912247776985168, 0.17778383195400238, 0.18115678429603577]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008008020813576877, "cos_r_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_innovation_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_apical_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_Ac_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "r_norm": [0.17753815650939941, 0.17617246508598328, 0.1796029806137085]}, {"step": 6800, "epoch": 14, "train_loss": 0.0007890297565609217, "cos_r_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_innovation_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_apical_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_Ac_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "r_norm": [0.18021561205387115, 0.1791214942932129, 0.18264533579349518]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005114716477692127, "cos_r_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_innovation_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_apical_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_Ac_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "r_norm": [0.17889344692230225, 0.17748130857944489, 0.18085458874702454]}, {"step": 7000, "epoch": 14, "train_loss": 0.0008774346788413823, "cos_r_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_innovation_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_apical_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_Ac_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "r_norm": [0.18395139276981354, 0.18258227407932281, 0.18588440120220184], "ldr": {"dL_sdil": -0.010152354836463928, "dL_bp": -0.023831740021705627, "ratio": 0.4260014093480921}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9743, "test_loss": 0.13829146881103516}], "final": {"test_acc": 0.9743, "test_loss": 0.13829146881103516, "wall_s": 21.884933948516846, "cos_r_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_innovation_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_apical_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_Ac_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_residual_s0.json b/results/nuis_ctrl_v1_rho0_residual_s0.json new file mode 100644 index 0000000..d2b537d --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_residual_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_residual_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3978137969970703, "cos_r_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_innovation_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_apical_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "cos_Ac_negg": [-0.022380873560905457, -0.021268675103783607, 0.034529902040958405], "r_norm": [4.8918256759643555, 4.819490432739258, 4.813833236694336], "ldr": {"dL_sdil": -0.007525444030761719, "dL_bp": -0.23199224472045898, "ratio": 0.03243834309991511}}, {"step": 100, "epoch": 0, "train_loss": 0.33160343766212463, "cos_r_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_innovation_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_apical_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "cos_Ac_negg": [0.4471272826194763, 0.3836216330528259, 0.3668625056743622], "r_norm": [0.8375285267829895, 0.818266749382019, 0.8186335563659668]}, {"step": 200, "epoch": 0, "train_loss": 0.29867634177207947, "cos_r_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_innovation_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_apical_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "cos_Ac_negg": [0.47979360818862915, 0.43690192699432373, 0.4357852041721344], "r_norm": [0.7242861390113831, 0.7113809585571289, 0.7089210152626038]}, {"step": 300, "epoch": 0, "train_loss": 0.28106310963630676, "cos_r_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_innovation_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_apical_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "cos_Ac_negg": [0.5084829926490784, 0.4854382872581482, 0.4583437442779541], "r_norm": [0.5968752503395081, 0.5882158875465393, 0.586020827293396]}, {"step": 400, "epoch": 0, "train_loss": 0.13037653267383575, "cos_r_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_innovation_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_apical_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "cos_Ac_negg": [0.5223413109779358, 0.49271804094314575, 0.44691815972328186], "r_norm": [0.5326493978500366, 0.5187801718711853, 0.5172048807144165]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9438, "test_loss": 0.19445072021484375}, {"step": 500, "epoch": 1, "train_loss": 0.3059273362159729, "cos_r_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_innovation_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_apical_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "cos_Ac_negg": [0.5512311458587646, 0.5073589086532593, 0.4986996650695801], "r_norm": [0.5493181943893433, 0.5408297777175903, 0.5343528985977173], "ldr": {"dL_sdil": -0.017139852046966553, "dL_bp": -0.0144614577293396, "ratio": 1.1852091516467935}}, {"step": 600, "epoch": 1, "train_loss": 0.2938694357872009, "cos_r_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_innovation_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_apical_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "cos_Ac_negg": [0.5305249094963074, 0.5065407752990723, 0.48065146803855896], "r_norm": [0.5047881007194519, 0.4929560720920563, 0.4911949336528778]}, {"step": 700, "epoch": 1, "train_loss": 0.0692189484834671, "cos_r_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_innovation_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_apical_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "cos_Ac_negg": [0.5255346298217773, 0.5136282444000244, 0.48031604290008545], "r_norm": [0.4124377369880676, 0.4063720703125, 0.40228134393692017]}, {"step": 800, "epoch": 1, "train_loss": 0.10150999575853348, "cos_r_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_innovation_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_apical_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "cos_Ac_negg": [0.5362330079078674, 0.5234052538871765, 0.4895923435688019], "r_norm": [0.41368532180786133, 0.408090204000473, 0.40460047125816345]}, {"step": 900, "epoch": 1, "train_loss": 0.08599556982517242, "cos_r_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_innovation_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_apical_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "cos_Ac_negg": [0.5581827759742737, 0.5421642065048218, 0.5054383277893066], "r_norm": [0.35195469856262207, 0.34207263588905334, 0.34272995591163635]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9534, "test_loss": 0.15320639877319336}, {"step": 1000, "epoch": 2, "train_loss": 0.19156473875045776, "cos_r_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_innovation_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_apical_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "cos_Ac_negg": [0.5704193711280823, 0.5684752464294434, 0.5213614702224731], "r_norm": [0.360003262758255, 0.3540140390396118, 0.3526112735271454], "ldr": {"dL_sdil": -0.009464025497436523, "dL_bp": -0.008196189999580383, "ratio": 1.1546859574901325}}, {"step": 1100, "epoch": 2, "train_loss": 0.10864664614200592, "cos_r_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_innovation_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_apical_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "cos_Ac_negg": [0.5639461278915405, 0.573844313621521, 0.5163298845291138], "r_norm": [0.28866949677467346, 0.2821740508079529, 0.28306305408477783]}, {"step": 1200, "epoch": 2, "train_loss": 0.031033102422952652, "cos_r_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_innovation_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_apical_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "cos_Ac_negg": [0.5674504637718201, 0.5838773250579834, 0.5309921503067017], "r_norm": [0.30560463666915894, 0.30166518688201904, 0.29685312509536743]}, {"step": 1300, "epoch": 2, "train_loss": 0.15016421675682068, "cos_r_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_innovation_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_apical_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "cos_Ac_negg": [0.5872721076011658, 0.5983583927154541, 0.5416527986526489], "r_norm": [0.279674768447876, 0.27491551637649536, 0.27148789167404175]}, {"step": 1400, "epoch": 2, "train_loss": 0.22877365350723267, "cos_r_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_innovation_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_apical_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "cos_Ac_negg": [0.5809815526008606, 0.6022815108299255, 0.5331538915634155], "r_norm": [0.29575562477111816, 0.2879781126976013, 0.28471633791923523]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9592, "test_loss": 0.13688001098632813}, {"step": 1500, "epoch": 3, "train_loss": 0.042368389666080475, "cos_r_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_innovation_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_apical_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "cos_Ac_negg": [0.5858829617500305, 0.6052732467651367, 0.5443239212036133], "r_norm": [0.3197419345378876, 0.3114323019981384, 0.3098006248474121], "ldr": {"dL_sdil": -0.008259907364845276, "dL_bp": -0.007200941443443298, "ratio": 1.1470593712945967}}, {"step": 1600, "epoch": 3, "train_loss": 0.05860002711415291, "cos_r_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_innovation_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_apical_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "cos_Ac_negg": [0.5875587463378906, 0.600779116153717, 0.5525841116905212], "r_norm": [0.30268725752830505, 0.29540443420410156, 0.2939055562019348]}, {"step": 1700, "epoch": 3, "train_loss": 0.08397884666919708, "cos_r_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_innovation_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_apical_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "cos_Ac_negg": [0.6034678816795349, 0.6009931564331055, 0.5679483413696289], "r_norm": [0.29046741127967834, 0.2826988697052002, 0.27999138832092285]}, {"step": 1800, "epoch": 3, "train_loss": 0.10174758732318878, "cos_r_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_innovation_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_apical_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "cos_Ac_negg": [0.6009317636489868, 0.6076779365539551, 0.563713788986206], "r_norm": [0.30012351274490356, 0.29224371910095215, 0.28972357511520386]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9665, "test_loss": 0.1125657865524292}, {"step": 1900, "epoch": 4, "train_loss": 0.07242363691329956, "cos_r_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_innovation_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_apical_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "cos_Ac_negg": [0.601470947265625, 0.6201019883155823, 0.5743652582168579], "r_norm": [0.28996407985687256, 0.28326937556266785, 0.2812224328517914]}, {"step": 2000, "epoch": 4, "train_loss": 0.06877899169921875, "cos_r_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_innovation_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_apical_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "cos_Ac_negg": [0.5867031812667847, 0.6191701889038086, 0.5793558359146118], "r_norm": [0.245533287525177, 0.24014459550380707, 0.23770779371261597], "ldr": {"dL_sdil": -0.006018199026584625, "dL_bp": -0.006462790071964264, "ratio": 0.9312075681820016}}, {"step": 2100, "epoch": 4, "train_loss": 0.05175407975912094, "cos_r_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_innovation_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_apical_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "cos_Ac_negg": [0.5937538146972656, 0.6262425184249878, 0.5848860144615173], "r_norm": [0.24031473696231842, 0.23528200387954712, 0.23227009177207947]}, {"step": 2200, "epoch": 4, "train_loss": 0.04244621843099594, "cos_r_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_innovation_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_apical_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "cos_Ac_negg": [0.5992727279663086, 0.6275991201400757, 0.6012266874313354], "r_norm": [0.2526210844516754, 0.24659496545791626, 0.24442610144615173]}, {"step": 2300, "epoch": 4, "train_loss": 0.12019313126802444, "cos_r_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_innovation_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_apical_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "cos_Ac_negg": [0.6200200319290161, 0.6389344930648804, 0.5988843441009521], "r_norm": [0.21065500378608704, 0.20645348727703094, 0.20339635014533997]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9679, "test_loss": 0.11181112232208253}, {"step": 2400, "epoch": 5, "train_loss": 0.033473387360572815, "cos_r_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_innovation_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_apical_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "cos_Ac_negg": [0.6181585788726807, 0.6414535045623779, 0.6110978722572327], "r_norm": [0.22075463831424713, 0.2167082130908966, 0.21338585019111633]}, {"step": 2500, "epoch": 5, "train_loss": 0.046647246927022934, "cos_r_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_innovation_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_apical_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "cos_Ac_negg": [0.6246686577796936, 0.6456069946289062, 0.6124918460845947], "r_norm": [0.23498451709747314, 0.2304421216249466, 0.22597557306289673], "ldr": {"dL_sdil": -0.00819263607263565, "dL_bp": -0.009024456143379211, "ratio": 0.9078260166011416}}, {"step": 2600, "epoch": 5, "train_loss": 0.023400278761982918, "cos_r_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_innovation_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_apical_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "cos_Ac_negg": [0.6155157089233398, 0.6328470706939697, 0.6116020679473877], "r_norm": [0.2576431632041931, 0.2516178488731384, 0.24756699800491333]}, {"step": 2700, "epoch": 5, "train_loss": 0.05335155129432678, "cos_r_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_innovation_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_apical_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "cos_Ac_negg": [0.6189650297164917, 0.6487292051315308, 0.612339437007904], "r_norm": [0.2206578552722931, 0.2165922373533249, 0.2133471816778183]}, {"step": 2800, "epoch": 5, "train_loss": 0.020739704370498657, "cos_r_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_innovation_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_apical_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "cos_Ac_negg": [0.6090052723884583, 0.6454235911369324, 0.6233847737312317], "r_norm": [0.2329060584306717, 0.2280796319246292, 0.22505390644073486]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9685, "test_loss": 0.11429850444793702}, {"step": 2900, "epoch": 6, "train_loss": 0.02710738405585289, "cos_r_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_innovation_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_apical_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "cos_Ac_negg": [0.6179582476615906, 0.6528295278549194, 0.624610960483551], "r_norm": [0.2212202548980713, 0.21703368425369263, 0.21472647786140442]}, {"step": 3000, "epoch": 6, "train_loss": 0.003210586030036211, "cos_r_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_innovation_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_apical_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "cos_Ac_negg": [0.613447368144989, 0.652495265007019, 0.6295528411865234], "r_norm": [0.19350673258304596, 0.1897171288728714, 0.18670257925987244], "ldr": {"dL_sdil": -0.007917709648609161, "dL_bp": -0.009828649461269379, "ratio": 0.805574527793423}}, {"step": 3100, "epoch": 6, "train_loss": 0.06387314200401306, "cos_r_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_innovation_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_apical_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "cos_Ac_negg": [0.6198863983154297, 0.6491948366165161, 0.6214192509651184], "r_norm": [0.17050178349018097, 0.16748245060443878, 0.16528522968292236]}, {"step": 3200, "epoch": 6, "train_loss": 0.04849023371934891, "cos_r_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_innovation_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_apical_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "cos_Ac_negg": [0.6148456335067749, 0.6510782241821289, 0.6248921155929565], "r_norm": [0.18880192935466766, 0.18442402780056, 0.18181553483009338]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9723, "test_loss": 0.11568929290771485}, {"step": 3300, "epoch": 7, "train_loss": 0.006756230723112822, "cos_r_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_innovation_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_apical_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "cos_Ac_negg": [0.6155461668968201, 0.6503416299819946, 0.6332229971885681], "r_norm": [0.1959608942270279, 0.19186583161354065, 0.1897565722465515]}, {"step": 3400, "epoch": 7, "train_loss": 0.005082777701318264, "cos_r_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_innovation_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_apical_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "cos_Ac_negg": [0.6128873825073242, 0.6470330953598022, 0.6373146176338196], "r_norm": [0.2011243999004364, 0.19720213115215302, 0.19440439343452454]}, {"step": 3500, "epoch": 7, "train_loss": 0.008085569366812706, "cos_r_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_innovation_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_apical_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "cos_Ac_negg": [0.6301254034042358, 0.6621131896972656, 0.645305335521698], "r_norm": [0.1824260950088501, 0.17886941134929657, 0.17662227153778076], "ldr": {"dL_sdil": -0.008172653615474701, "dL_bp": -0.011752061545848846, "ratio": 0.6954229760957564}}, {"step": 3600, "epoch": 7, "train_loss": 0.009758161380887032, "cos_r_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_innovation_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_apical_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "cos_Ac_negg": [0.6344751715660095, 0.6549732685089111, 0.6478409767150879], "r_norm": [0.15041005611419678, 0.1479288935661316, 0.1454424113035202]}, {"step": 3700, "epoch": 7, "train_loss": 0.005388827063143253, "cos_r_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_innovation_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_apical_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "cos_Ac_negg": [0.6371345520019531, 0.6556147336959839, 0.6490602493286133], "r_norm": [0.1920110434293747, 0.18857887387275696, 0.185125932097435]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9706, "test_loss": 0.12330467185974121}, {"step": 3800, "epoch": 8, "train_loss": 0.056129325181245804, "cos_r_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_innovation_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_apical_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "cos_Ac_negg": [0.6330610513687134, 0.6584164500236511, 0.6486068964004517], "r_norm": [0.1895904839038849, 0.18607723712921143, 0.18282277882099152]}, {"step": 3900, "epoch": 8, "train_loss": 0.011785263195633888, "cos_r_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_innovation_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_apical_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "cos_Ac_negg": [0.6330327391624451, 0.6584503650665283, 0.649254322052002], "r_norm": [0.17273201048374176, 0.16941824555397034, 0.1666167974472046]}, {"step": 4000, "epoch": 8, "train_loss": 0.000689225154928863, "cos_r_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_innovation_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_apical_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "cos_Ac_negg": [0.6296203136444092, 0.6526379585266113, 0.6534024477005005], "r_norm": [0.17630384862422943, 0.17299006879329681, 0.17047850787639618], "ldr": {"dL_sdil": -0.009058646857738495, "dL_bp": -0.014096729457378387, "ratio": 0.6426062786497684}}, {"step": 4100, "epoch": 8, "train_loss": 0.03716294467449188, "cos_r_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_innovation_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_apical_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "cos_Ac_negg": [0.6289478540420532, 0.655285120010376, 0.6590647101402283], "r_norm": [0.1756715625524521, 0.17255355417728424, 0.16945818066596985]}, {"step": 4200, "epoch": 8, "train_loss": 0.007642740849405527, "cos_r_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_innovation_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_apical_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "cos_Ac_negg": [0.6331797242164612, 0.657547116279602, 0.66251140832901], "r_norm": [0.17260634899139404, 0.16935701668262482, 0.1664430946111679]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9715, "test_loss": 0.12795383539199828}, {"step": 4300, "epoch": 9, "train_loss": 0.005126616917550564, "cos_r_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_innovation_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_apical_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "cos_Ac_negg": [0.6217033863067627, 0.6551398038864136, 0.6618025302886963], "r_norm": [0.19533851742744446, 0.19135543704032898, 0.1880948841571808]}, {"step": 4400, "epoch": 9, "train_loss": 0.003296743845567107, "cos_r_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_innovation_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_apical_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "cos_Ac_negg": [0.6174290776252747, 0.6536604166030884, 0.6603547930717468], "r_norm": [0.18917936086654663, 0.1854909360408783, 0.18246984481811523]}, {"step": 4500, "epoch": 9, "train_loss": 0.0014693234115839005, "cos_r_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_innovation_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_apical_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "cos_Ac_negg": [0.6212970018386841, 0.6520422697067261, 0.6648234128952026], "r_norm": [0.17914845049381256, 0.17544743418693542, 0.17231427133083344], "ldr": {"dL_sdil": -0.00979585200548172, "dL_bp": -0.01655876636505127, "ratio": 0.5915810266009143}}, {"step": 4600, "epoch": 9, "train_loss": 0.032596148550510406, "cos_r_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_innovation_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_apical_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "cos_Ac_negg": [0.6234061121940613, 0.6518699526786804, 0.6677985191345215], "r_norm": [0.19013909995555878, 0.1861753761768341, 0.1826411783695221]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.974, "test_loss": 0.12575695457458497}, {"step": 4700, "epoch": 10, "train_loss": 0.0034915334545075893, "cos_r_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_innovation_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_apical_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "cos_Ac_negg": [0.6199545860290527, 0.6492616534233093, 0.6606061458587646], "r_norm": [0.18801575899124146, 0.1843785047531128, 0.18106210231781006]}, {"step": 4800, "epoch": 10, "train_loss": 0.0008336055907420814, "cos_r_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_innovation_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_apical_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "cos_Ac_negg": [0.6216712594032288, 0.6511514782905579, 0.6634930968284607], "r_norm": [0.1824166178703308, 0.17903029918670654, 0.17574921250343323]}, {"step": 4900, "epoch": 10, "train_loss": 0.001636673929169774, "cos_r_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_innovation_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_apical_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "cos_Ac_negg": [0.6146307587623596, 0.6468146443367004, 0.664393424987793], "r_norm": [0.17372174561023712, 0.17035624384880066, 0.16707497835159302]}, {"step": 5000, "epoch": 10, "train_loss": 0.0015556986909359694, "cos_r_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_innovation_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_apical_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "cos_Ac_negg": [0.6180275678634644, 0.6454173922538757, 0.6625872850418091], "r_norm": [0.17774055898189545, 0.1746811866760254, 0.1715075671672821], "ldr": {"dL_sdil": -0.010479673743247986, "dL_bp": -0.01817304641008377, "ratio": 0.5766602641499378}}, {"step": 5100, "epoch": 10, "train_loss": 0.0015227128751575947, "cos_r_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_innovation_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_apical_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "cos_Ac_negg": [0.6159430742263794, 0.6451226472854614, 0.6630815267562866], "r_norm": [0.18908870220184326, 0.18590903282165527, 0.18229123950004578]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9733, "test_loss": 0.12908906383514404}, {"step": 5200, "epoch": 11, "train_loss": 0.001309263869188726, "cos_r_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_innovation_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_apical_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "cos_Ac_negg": [0.614648699760437, 0.6446752548217773, 0.665759265422821], "r_norm": [0.17493118345737457, 0.17182868719100952, 0.16859647631645203]}, {"step": 5300, "epoch": 11, "train_loss": 0.0011599198915064335, "cos_r_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_innovation_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_apical_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "cos_Ac_negg": [0.6132922172546387, 0.6421023607254028, 0.6631933450698853], "r_norm": [0.16871681809425354, 0.16564862430095673, 0.16264042258262634]}, {"step": 5400, "epoch": 11, "train_loss": 0.0014726945664733648, "cos_r_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_innovation_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_apical_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "cos_Ac_negg": [0.6104533672332764, 0.6421495079994202, 0.6644812822341919], "r_norm": [0.1738046109676361, 0.17061923444271088, 0.167450949549675]}, {"step": 5500, "epoch": 11, "train_loss": 0.0022923252545297146, "cos_r_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_innovation_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_apical_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "cos_Ac_negg": [0.6091525554656982, 0.6400614976882935, 0.6668003797531128], "r_norm": [0.1730632185935974, 0.1701216697692871, 0.16676318645477295], "ldr": {"dL_sdil": -0.01092517375946045, "dL_bp": -0.019617542624473572, "ratio": 0.5569083737241846}}, {"step": 5600, "epoch": 11, "train_loss": 0.003013922367244959, "cos_r_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_innovation_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_apical_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "cos_Ac_negg": [0.6125717163085938, 0.6437373161315918, 0.6673654317855835], "r_norm": [0.17643772065639496, 0.17336799204349518, 0.16997528076171875]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.974, "test_loss": 0.1334169048309326}, {"step": 5700, "epoch": 12, "train_loss": 0.00026223674649372697, "cos_r_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_innovation_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_apical_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "cos_Ac_negg": [0.6107821464538574, 0.6430332660675049, 0.6681469082832336], "r_norm": [0.1758565604686737, 0.17264872789382935, 0.16945300996303558]}, {"step": 5800, "epoch": 12, "train_loss": 0.0003017765120603144, "cos_r_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_innovation_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_apical_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "cos_Ac_negg": [0.6081430315971375, 0.6421799659729004, 0.6661195755004883], "r_norm": [0.17203187942504883, 0.1690245270729065, 0.16582567989826202]}, {"step": 5900, "epoch": 12, "train_loss": 0.0013039933983236551, "cos_r_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_innovation_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_apical_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "cos_Ac_negg": [0.6071442365646362, 0.6420730352401733, 0.6674133539199829], "r_norm": [0.17366287112236023, 0.17064981162548065, 0.16749976575374603]}, {"step": 6000, "epoch": 12, "train_loss": 0.0008040090324357152, "cos_r_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_innovation_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_apical_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "cos_Ac_negg": [0.6076933145523071, 0.6441484093666077, 0.6691628694534302], "r_norm": [0.17631205916404724, 0.17321062088012695, 0.17003434896469116], "ldr": {"dL_sdil": -0.011585041880607605, "dL_bp": -0.022001802921295166, "ratio": 0.526549661500451}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9745, "test_loss": 0.1352493793487549}, {"step": 6100, "epoch": 13, "train_loss": 0.0014463819097727537, "cos_r_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_innovation_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_apical_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "cos_Ac_negg": [0.6067887544631958, 0.6430466175079346, 0.6696619987487793], "r_norm": [0.17908143997192383, 0.17593857645988464, 0.17247158288955688]}, {"step": 6200, "epoch": 13, "train_loss": 0.0002795365871861577, "cos_r_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_innovation_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_apical_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "cos_Ac_negg": [0.6057158708572388, 0.6432228088378906, 0.6696715354919434], "r_norm": [0.17346340417861938, 0.17047497630119324, 0.16715440154075623]}, {"step": 6300, "epoch": 13, "train_loss": 0.0003245513653382659, "cos_r_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_innovation_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_apical_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "cos_Ac_negg": [0.6048141717910767, 0.6426535248756409, 0.6699920892715454], "r_norm": [0.1769692301750183, 0.17405544221401215, 0.17055493593215942]}, {"step": 6400, "epoch": 13, "train_loss": 0.0006700620288029313, "cos_r_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_innovation_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_apical_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "cos_Ac_negg": [0.6028282642364502, 0.6402886509895325, 0.6704275012016296], "r_norm": [0.17150437831878662, 0.16849254071712494, 0.16540826857089996]}, {"step": 6500, "epoch": 13, "train_loss": 0.00013033303548581898, "cos_r_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_innovation_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_apical_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "cos_Ac_negg": [0.6023449897766113, 0.6430495381355286, 0.6708477735519409], "r_norm": [0.16887250542640686, 0.1660538762807846, 0.16295760869979858], "ldr": {"dL_sdil": -0.011667512357234955, "dL_bp": -0.022730067372322083, "ratio": 0.5133074251879357}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9737, "test_loss": 0.13770540838241577}, {"step": 6600, "epoch": 14, "train_loss": 0.002334980759769678, "cos_r_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_innovation_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_apical_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "cos_Ac_negg": [0.6029652953147888, 0.6421791315078735, 0.6708009243011475], "r_norm": [0.1712503284215927, 0.16834032535552979, 0.16512611508369446]}, {"step": 6700, "epoch": 14, "train_loss": 0.0007730001234449446, "cos_r_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_innovation_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_apical_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "cos_Ac_negg": [0.6041147708892822, 0.6437222957611084, 0.6716139912605286], "r_norm": [0.17286744713783264, 0.16990327835083008, 0.1666618287563324]}, {"step": 6800, "epoch": 14, "train_loss": 0.0018118484877049923, "cos_r_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_innovation_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_apical_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "cos_Ac_negg": [0.6043601632118225, 0.6438187956809998, 0.6724725961685181], "r_norm": [0.17235802114009857, 0.16948559880256653, 0.166168212890625]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005928482860326767, "cos_r_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_innovation_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_apical_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "cos_Ac_negg": [0.605883777141571, 0.6448389887809753, 0.6733330488204956], "r_norm": [0.17221295833587646, 0.16921906173229218, 0.16597718000411987]}, {"step": 7000, "epoch": 14, "train_loss": 0.000632595329079777, "cos_r_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_innovation_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_apical_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "cos_Ac_negg": [0.6015108823776245, 0.6416085958480835, 0.6718443632125854], "r_norm": [0.1718350499868393, 0.168907031416893, 0.16570332646369934], "ldr": {"dL_sdil": -0.012075692415237427, "dL_bp": -0.023587502539157867, "ratio": 0.5119529884603272}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9745, "test_loss": 0.14063884963989257}], "final": {"test_acc": 0.9745, "test_loss": 0.14063884963989257, "wall_s": 20.881760358810425, "cos_r_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_innovation_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_apical_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058], "cos_Ac_negg": [0.6034194231033325, 0.6417666077613831, 0.6718880534172058]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_residual_s1.json b/results/nuis_ctrl_v1_rho0_residual_s1.json new file mode 100644 index 0000000..48d36e7 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_residual_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_residual_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.247864246368408, "cos_r_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_innovation_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_apical_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "cos_Ac_negg": [0.012752903625369072, -0.010880010202527046, -0.004258741624653339], "r_norm": [4.729090213775635, 4.788191795349121, 4.781856536865234], "ldr": {"dL_sdil": -0.035402774810791016, "dL_bp": -0.2081589698791504, "ratio": 0.17007566299614468}}, {"step": 100, "epoch": 0, "train_loss": 0.3791123926639557, "cos_r_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_innovation_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_apical_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "cos_Ac_negg": [0.4325192868709564, 0.3425039052963257, 0.34370851516723633], "r_norm": [0.8098099231719971, 0.8149547576904297, 0.8344215154647827]}, {"step": 200, "epoch": 0, "train_loss": 0.2855910360813141, "cos_r_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_innovation_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_apical_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "cos_Ac_negg": [0.47700124979019165, 0.42365050315856934, 0.39592015743255615], "r_norm": [0.7800662517547607, 0.7833849787712097, 0.7984472513198853]}, {"step": 300, "epoch": 0, "train_loss": 0.3106156587600708, "cos_r_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_innovation_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_apical_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "cos_Ac_negg": [0.49683982133865356, 0.4656694531440735, 0.4201711118221283], "r_norm": [0.6266008019447327, 0.6296441555023193, 0.6423605680465698]}, {"step": 400, "epoch": 0, "train_loss": 0.1368972212076187, "cos_r_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_innovation_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_apical_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "cos_Ac_negg": [0.5131005048751831, 0.48043298721313477, 0.4115670323371887], "r_norm": [0.4948488473892212, 0.49738022685050964, 0.5028847455978394]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9471, "test_loss": 0.18460173034667968}, {"step": 500, "epoch": 1, "train_loss": 0.22857800126075745, "cos_r_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_innovation_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_apical_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "cos_Ac_negg": [0.5339407920837402, 0.4849788546562195, 0.4415796399116516], "r_norm": [0.5006556510925293, 0.501956582069397, 0.5072718858718872], "ldr": {"dL_sdil": -0.010978296399116516, "dL_bp": -0.00958406925201416, "ratio": 1.1454734007487841}}, {"step": 600, "epoch": 1, "train_loss": 0.26168546080589294, "cos_r_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_innovation_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_apical_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "cos_Ac_negg": [0.521554708480835, 0.49236857891082764, 0.46233895421028137], "r_norm": [0.4931199550628662, 0.49525341391563416, 0.5031353235244751]}, {"step": 700, "epoch": 1, "train_loss": 0.08662884682416916, "cos_r_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_innovation_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_apical_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "cos_Ac_negg": [0.52215576171875, 0.4998355507850647, 0.4687618613243103], "r_norm": [0.41211724281311035, 0.4123530983924866, 0.42171144485473633]}, {"step": 800, "epoch": 1, "train_loss": 0.06552650779485703, "cos_r_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_innovation_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_apical_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "cos_Ac_negg": [0.5319322347640991, 0.5127753019332886, 0.47523486614227295], "r_norm": [0.37752482295036316, 0.37500566244125366, 0.38382813334465027]}, {"step": 900, "epoch": 1, "train_loss": 0.16771306097507477, "cos_r_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_innovation_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_apical_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "cos_Ac_negg": [0.5413532853126526, 0.5249296426773071, 0.47342005372047424], "r_norm": [0.3334032893180847, 0.33176112174987793, 0.3353516459465027]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9596, "test_loss": 0.1381149112701416}, {"step": 1000, "epoch": 2, "train_loss": 0.0976904109120369, "cos_r_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_innovation_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_apical_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "cos_Ac_negg": [0.5671747922897339, 0.5522836446762085, 0.4741804599761963], "r_norm": [0.3129379451274872, 0.31030431389808655, 0.3134283423423767], "ldr": {"dL_sdil": -0.005524396896362305, "dL_bp": -0.00501541793346405, "ratio": 1.101482861378755}}, {"step": 1100, "epoch": 2, "train_loss": 0.11463465541601181, "cos_r_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_innovation_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_apical_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "cos_Ac_negg": [0.5700598955154419, 0.5434622168540955, 0.49277806282043457], "r_norm": [0.27226677536964417, 0.27177441120147705, 0.2737849950790405]}, {"step": 1200, "epoch": 2, "train_loss": 0.022953230887651443, "cos_r_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_innovation_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_apical_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "cos_Ac_negg": [0.5559427738189697, 0.5386655330657959, 0.4789605736732483], "r_norm": [0.3118118643760681, 0.31061288714408875, 0.3118583559989929]}, {"step": 1300, "epoch": 2, "train_loss": 0.19519557058811188, "cos_r_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_innovation_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_apical_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "cos_Ac_negg": [0.5691392421722412, 0.5552347898483276, 0.499505877494812], "r_norm": [0.27181971073150635, 0.26876047253608704, 0.2751275300979614]}, {"step": 1400, "epoch": 2, "train_loss": 0.20355364680290222, "cos_r_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_innovation_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_apical_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "cos_Ac_negg": [0.5666496753692627, 0.5699707865715027, 0.5015599131584167], "r_norm": [0.26793360710144043, 0.2665303349494934, 0.2706131637096405]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9619, "test_loss": 0.12944196891784668}, {"step": 1500, "epoch": 3, "train_loss": 0.04727518558502197, "cos_r_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_innovation_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_apical_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "cos_Ac_negg": [0.5635039806365967, 0.5683403611183167, 0.5114924907684326], "r_norm": [0.2907962203025818, 0.2895806133747101, 0.2904374301433563], "ldr": {"dL_sdil": -0.00766473263502121, "dL_bp": -0.007067129015922546, "ratio": 1.0845610173172495}}, {"step": 1600, "epoch": 3, "train_loss": 0.028240162879228592, "cos_r_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_innovation_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_apical_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "cos_Ac_negg": [0.5836167931556702, 0.5681452751159668, 0.5083830952644348], "r_norm": [0.24258095026016235, 0.2405192106962204, 0.2429429292678833]}, {"step": 1700, "epoch": 3, "train_loss": 0.08582475781440735, "cos_r_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_innovation_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_apical_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "cos_Ac_negg": [0.5944448709487915, 0.5831553936004639, 0.5183581709861755], "r_norm": [0.23459598422050476, 0.23319926857948303, 0.23500296473503113]}, {"step": 1800, "epoch": 3, "train_loss": 0.054279766976833344, "cos_r_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_innovation_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_apical_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "cos_Ac_negg": [0.5952188968658447, 0.5937332510948181, 0.5194491744041443], "r_norm": [0.2751310467720032, 0.2752900719642639, 0.2777182459831238]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9686, "test_loss": 0.1130959093093872}, {"step": 1900, "epoch": 4, "train_loss": 0.08464933186769485, "cos_r_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_innovation_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_apical_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "cos_Ac_negg": [0.5940946340560913, 0.5953451991081238, 0.5435026288032532], "r_norm": [0.21713416278362274, 0.21652851998806, 0.21772976219654083]}, {"step": 2000, "epoch": 4, "train_loss": 0.08816886693239212, "cos_r_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_innovation_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_apical_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "cos_Ac_negg": [0.6078377366065979, 0.599784255027771, 0.5560770034790039], "r_norm": [0.20291899144649506, 0.2031886875629425, 0.20419245958328247], "ldr": {"dL_sdil": -0.004766151309013367, "dL_bp": -0.005502060055732727, "ratio": 0.8662485070564434}}, {"step": 2100, "epoch": 4, "train_loss": 0.07242057472467422, "cos_r_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_innovation_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_apical_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "cos_Ac_negg": [0.5998598337173462, 0.609147846698761, 0.5555221438407898], "r_norm": [0.23937690258026123, 0.23910373449325562, 0.2406611144542694]}, {"step": 2200, "epoch": 4, "train_loss": 0.07056484371423721, "cos_r_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_innovation_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_apical_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "cos_Ac_negg": [0.5931302309036255, 0.5939610004425049, 0.5502951145172119], "r_norm": [0.23531854152679443, 0.23204778134822845, 0.23232631385326385]}, {"step": 2300, "epoch": 4, "train_loss": 0.12437652051448822, "cos_r_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_innovation_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_apical_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "cos_Ac_negg": [0.5943087935447693, 0.6049081683158875, 0.5639764666557312], "r_norm": [0.21436047554016113, 0.21310736238956451, 0.21543720364570618]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.967, "test_loss": 0.11917487754821778}, {"step": 2400, "epoch": 5, "train_loss": 0.02945767343044281, "cos_r_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_innovation_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_apical_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "cos_Ac_negg": [0.5876122713088989, 0.6044968366622925, 0.5625086426734924], "r_norm": [0.17013323307037354, 0.16899538040161133, 0.17079725861549377]}, {"step": 2500, "epoch": 5, "train_loss": 0.05113217979669571, "cos_r_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_innovation_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_apical_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "cos_Ac_negg": [0.6018557548522949, 0.6268559694290161, 0.5786784887313843], "r_norm": [0.19201859831809998, 0.190620556473732, 0.19072513282299042], "ldr": {"dL_sdil": -0.0062797218561172485, "dL_bp": -0.007466904819011688, "ratio": 0.8410073528898184}}, {"step": 2600, "epoch": 5, "train_loss": 0.00991781149059534, "cos_r_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_innovation_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_apical_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "cos_Ac_negg": [0.597125768661499, 0.6248884201049805, 0.5796911716461182], "r_norm": [0.19189369678497314, 0.1880035698413849, 0.18973243236541748]}, {"step": 2700, "epoch": 5, "train_loss": 0.05675402656197548, "cos_r_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_innovation_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_apical_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "cos_Ac_negg": [0.5978406667709351, 0.6243821382522583, 0.5874540209770203], "r_norm": [0.2130669355392456, 0.20993337035179138, 0.21378093957901]}, {"step": 2800, "epoch": 5, "train_loss": 0.009208951145410538, "cos_r_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_innovation_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_apical_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "cos_Ac_negg": [0.5927330255508423, 0.6231971979141235, 0.5867453813552856], "r_norm": [0.21457375586032867, 0.21203386783599854, 0.21387223899364471]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9724, "test_loss": 0.10937111053466797}, {"step": 2900, "epoch": 6, "train_loss": 0.014543101191520691, "cos_r_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_innovation_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_apical_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "cos_Ac_negg": [0.6086075305938721, 0.6244409084320068, 0.5985665321350098], "r_norm": [0.18077343702316284, 0.1798022985458374, 0.18212389945983887]}, {"step": 3000, "epoch": 6, "train_loss": 0.006073781289160252, "cos_r_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_innovation_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_apical_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "cos_Ac_negg": [0.5966978073120117, 0.621256411075592, 0.6010026931762695], "r_norm": [0.15268780291080475, 0.15006791055202484, 0.1529131680727005], "ldr": {"dL_sdil": -0.005284003913402557, "dL_bp": -0.006912320852279663, "ratio": 0.7644326741082784}}, {"step": 3100, "epoch": 6, "train_loss": 0.07234150171279907, "cos_r_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_innovation_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_apical_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "cos_Ac_negg": [0.6114012598991394, 0.6296575665473938, 0.6066649556159973], "r_norm": [0.13482704758644104, 0.13283531367778778, 0.1336984634399414]}, {"step": 3200, "epoch": 6, "train_loss": 0.06304893642663956, "cos_r_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_innovation_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_apical_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "cos_Ac_negg": [0.5907710790634155, 0.6187146902084351, 0.611295759677887], "r_norm": [0.18191789090633392, 0.1812913715839386, 0.18045282363891602]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9716, "test_loss": 0.11546330757141113}, {"step": 3300, "epoch": 7, "train_loss": 0.008135353215038776, "cos_r_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_innovation_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_apical_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "cos_Ac_negg": [0.605095624923706, 0.6289936304092407, 0.6129757165908813], "r_norm": [0.1324104368686676, 0.13133256137371063, 0.1319148689508438]}, {"step": 3400, "epoch": 7, "train_loss": 0.003062707372009754, "cos_r_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_innovation_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_apical_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "cos_Ac_negg": [0.609114944934845, 0.6304043531417847, 0.6218795776367188], "r_norm": [0.17751097679138184, 0.17610973119735718, 0.17820629477500916]}, {"step": 3500, "epoch": 7, "train_loss": 0.0075003234669566154, "cos_r_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_innovation_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_apical_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "cos_Ac_negg": [0.6195396184921265, 0.644755482673645, 0.6330949068069458], "r_norm": [0.15693698823451996, 0.1564456969499588, 0.15661953389644623], "ldr": {"dL_sdil": -0.005416072905063629, "dL_bp": -0.0077264681458473206, "ratio": 0.7009765397110399}}, {"step": 3600, "epoch": 7, "train_loss": 0.004378090146929026, "cos_r_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_innovation_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_apical_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "cos_Ac_negg": [0.6075403690338135, 0.6391722559928894, 0.6257597208023071], "r_norm": [0.15218180418014526, 0.15111035108566284, 0.1530647873878479]}, {"step": 3700, "epoch": 7, "train_loss": 0.005401092581450939, "cos_r_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_innovation_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_apical_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "cos_Ac_negg": [0.6035904884338379, 0.6418662667274475, 0.6278824806213379], "r_norm": [0.15374818444252014, 0.1511906236410141, 0.15268918871879578]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9725, "test_loss": 0.12304098224639892}, {"step": 3800, "epoch": 8, "train_loss": 0.020612072199583054, "cos_r_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_innovation_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_apical_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "cos_Ac_negg": [0.6025469303131104, 0.6354708075523376, 0.6354650259017944], "r_norm": [0.18328630924224854, 0.18096177279949188, 0.1839454621076584]}, {"step": 3900, "epoch": 8, "train_loss": 0.00907292403280735, "cos_r_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_innovation_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_apical_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "cos_Ac_negg": [0.6152520775794983, 0.6486815214157104, 0.6391942501068115], "r_norm": [0.15745952725410461, 0.15553179383277893, 0.15759898722171783]}, {"step": 4000, "epoch": 8, "train_loss": 0.0011170797515660524, "cos_r_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_innovation_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_apical_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "cos_Ac_negg": [0.608879029750824, 0.6432069540023804, 0.6362855434417725], "r_norm": [0.13805150985717773, 0.13616320490837097, 0.1368236541748047], "ldr": {"dL_sdil": -0.005435429513454437, "dL_bp": -0.00862833857536316, "ratio": 0.6299508840525146}}, {"step": 4100, "epoch": 8, "train_loss": 0.06657039374113083, "cos_r_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_innovation_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_apical_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "cos_Ac_negg": [0.6083284020423889, 0.6368762254714966, 0.6377155780792236], "r_norm": [0.14812925457954407, 0.14660516381263733, 0.14825358986854553]}, {"step": 4200, "epoch": 8, "train_loss": 0.01757517084479332, "cos_r_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_innovation_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_apical_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "cos_Ac_negg": [0.6107895970344543, 0.6481828093528748, 0.6390138268470764], "r_norm": [0.14519406855106354, 0.14317519962787628, 0.1447223722934723]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9733, "test_loss": 0.12583541717529298}, {"step": 4300, "epoch": 9, "train_loss": 0.0043710931204259396, "cos_r_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_innovation_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_apical_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "cos_Ac_negg": [0.6124082803726196, 0.6420118808746338, 0.6461033821105957], "r_norm": [0.17884290218353271, 0.1771182417869568, 0.17936065793037415]}, {"step": 4400, "epoch": 9, "train_loss": 0.012221035547554493, "cos_r_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_innovation_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_apical_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "cos_Ac_negg": [0.6140802502632141, 0.6452147960662842, 0.6478192806243896], "r_norm": [0.159407377243042, 0.15782558917999268, 0.15985700488090515]}, {"step": 4500, "epoch": 9, "train_loss": 0.004442101344466209, "cos_r_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_innovation_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_apical_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "cos_Ac_negg": [0.6168966889381409, 0.645344614982605, 0.6513071060180664], "r_norm": [0.1378084421157837, 0.13638418912887573, 0.13881981372833252], "ldr": {"dL_sdil": -0.006090611219406128, "dL_bp": -0.009796351194381714, "ratio": 0.6217224248656115}}, {"step": 4600, "epoch": 9, "train_loss": 0.05368257686495781, "cos_r_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_innovation_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_apical_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "cos_Ac_negg": [0.6151841878890991, 0.647202730178833, 0.6508036851882935], "r_norm": [0.14086458086967468, 0.1384350061416626, 0.1409018635749817]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9722, "test_loss": 0.1313097234725952}, {"step": 4700, "epoch": 10, "train_loss": 0.0048868609592318535, "cos_r_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_innovation_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_apical_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "cos_Ac_negg": [0.5991007089614868, 0.6374084949493408, 0.652641236782074], "r_norm": [0.13858553767204285, 0.13714627921581268, 0.13974396884441376]}, {"step": 4800, "epoch": 10, "train_loss": 0.001804178231395781, "cos_r_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_innovation_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_apical_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "cos_Ac_negg": [0.6069160103797913, 0.6433519124984741, 0.6545641422271729], "r_norm": [0.1522747427225113, 0.1504199206829071, 0.15280333161354065]}, {"step": 4900, "epoch": 10, "train_loss": 0.0028787143528461456, "cos_r_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_innovation_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_apical_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "cos_Ac_negg": [0.6024950742721558, 0.6423514485359192, 0.6571534872055054], "r_norm": [0.1442190408706665, 0.14268440008163452, 0.14490017294883728]}, {"step": 5000, "epoch": 10, "train_loss": 0.0013339570723474026, "cos_r_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_innovation_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_apical_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "cos_Ac_negg": [0.6024949550628662, 0.6410781145095825, 0.6558099389076233], "r_norm": [0.1472242772579193, 0.14609915018081665, 0.14829722046852112], "ldr": {"dL_sdil": -0.006713069975376129, "dL_bp": -0.011234067380428314, "ratio": 0.5975636203741715}}, {"step": 5100, "epoch": 10, "train_loss": 0.0018756217323243618, "cos_r_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_innovation_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_apical_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "cos_Ac_negg": [0.596645712852478, 0.6321779489517212, 0.655342698097229], "r_norm": [0.14868545532226562, 0.14725178480148315, 0.14963632822036743]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.973, "test_loss": 0.13351503677368165}, {"step": 5200, "epoch": 11, "train_loss": 0.0014711369294673204, "cos_r_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_innovation_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_apical_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "cos_Ac_negg": [0.597823977470398, 0.6412510275840759, 0.6565788984298706], "r_norm": [0.12346100807189941, 0.12253932654857635, 0.12461203336715698]}, {"step": 5300, "epoch": 11, "train_loss": 0.001629142090678215, "cos_r_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_innovation_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_apical_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "cos_Ac_negg": [0.5951378345489502, 0.6368370652198792, 0.6564886569976807], "r_norm": [0.12968139350414276, 0.1281958818435669, 0.13058608770370483]}, {"step": 5400, "epoch": 11, "train_loss": 0.0024097473360598087, "cos_r_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_innovation_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_apical_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "cos_Ac_negg": [0.5949071645736694, 0.6371099352836609, 0.6593981981277466], "r_norm": [0.12974487245082855, 0.12847599387168884, 0.13078004121780396]}, {"step": 5500, "epoch": 11, "train_loss": 0.0010489317355677485, "cos_r_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_innovation_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_apical_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "cos_Ac_negg": [0.5930535197257996, 0.6363919973373413, 0.659381091594696], "r_norm": [0.13474895060062408, 0.13274598121643066, 0.13516360521316528], "ldr": {"dL_sdil": -0.006466925144195557, "dL_bp": -0.01084946095943451, "ratio": 0.5960595801372074}}, {"step": 5600, "epoch": 11, "train_loss": 0.0009077594731934369, "cos_r_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_innovation_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_apical_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "cos_Ac_negg": [0.5962475538253784, 0.6397339105606079, 0.6600366830825806], "r_norm": [0.13371193408966064, 0.13146626949310303, 0.13360051810741425]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9739, "test_loss": 0.1354274552345276}, {"step": 5700, "epoch": 12, "train_loss": 0.0005193198448978364, "cos_r_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_innovation_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_apical_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "cos_Ac_negg": [0.5935052633285522, 0.6346560716629028, 0.6568100452423096], "r_norm": [0.13641861081123352, 0.13459017872810364, 0.1368558406829834]}, {"step": 5800, "epoch": 12, "train_loss": 0.0007246336317621171, "cos_r_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_innovation_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_apical_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "cos_Ac_negg": [0.5907622575759888, 0.6327726244926453, 0.6565173864364624], "r_norm": [0.1397680640220642, 0.137950599193573, 0.14015179872512817]}, {"step": 5900, "epoch": 12, "train_loss": 0.0012115520657971501, "cos_r_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_innovation_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_apical_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "cos_Ac_negg": [0.5903595685958862, 0.6344199180603027, 0.658551037311554], "r_norm": [0.12886938452720642, 0.12738792598247528, 0.1296568214893341]}, {"step": 6000, "epoch": 12, "train_loss": 0.0011408575810492039, "cos_r_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_innovation_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_apical_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "cos_Ac_negg": [0.5883030891418457, 0.6310743093490601, 0.6591668725013733], "r_norm": [0.13654574751853943, 0.13504381477832794, 0.1375402808189392], "ldr": {"dL_sdil": -0.006541714072227478, "dL_bp": -0.010865777730941772, "ratio": 0.602047477337868}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9739, "test_loss": 0.13652948637008666}, {"step": 6100, "epoch": 13, "train_loss": 0.0016530295833945274, "cos_r_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_innovation_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_apical_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "cos_Ac_negg": [0.5924124717712402, 0.6343724131584167, 0.6603498458862305], "r_norm": [0.13538393378257751, 0.13388687372207642, 0.1362152248620987]}, {"step": 6200, "epoch": 13, "train_loss": 0.0004883500514551997, "cos_r_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_innovation_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_apical_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "cos_Ac_negg": [0.5938141345977783, 0.6332895755767822, 0.6610090136528015], "r_norm": [0.13575664162635803, 0.13420100510120392, 0.13645632565021515]}, {"step": 6300, "epoch": 13, "train_loss": 0.0005272991256788373, "cos_r_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_innovation_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_apical_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "cos_Ac_negg": [0.588531494140625, 0.6321733593940735, 0.6613149046897888], "r_norm": [0.1379033327102661, 0.1362687349319458, 0.13877828419208527]}, {"step": 6400, "epoch": 13, "train_loss": 0.0008385080145671964, "cos_r_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_innovation_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_apical_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "cos_Ac_negg": [0.5887690782546997, 0.6329339742660522, 0.6627668142318726], "r_norm": [0.1363997906446457, 0.1347801685333252, 0.13696125149726868]}, {"step": 6500, "epoch": 13, "train_loss": 0.0004538414941634983, "cos_r_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_innovation_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_apical_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "cos_Ac_negg": [0.5886293649673462, 0.6323056221008301, 0.6620996594429016], "r_norm": [0.13410302996635437, 0.13246551156044006, 0.1348181515932083], "ldr": {"dL_sdil": -0.006781101226806641, "dL_bp": -0.01180441677570343, "ratio": 0.5744545754063781}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9737, "test_loss": 0.13901660509109498}, {"step": 6600, "epoch": 14, "train_loss": 0.0003573036228772253, "cos_r_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_innovation_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_apical_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "cos_Ac_negg": [0.5871461033821106, 0.6295948028564453, 0.6612600088119507], "r_norm": [0.13315507769584656, 0.13159938156604767, 0.1339610517024994]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008824412361718714, "cos_r_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_innovation_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_apical_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "cos_Ac_negg": [0.588778018951416, 0.6299223303794861, 0.6623358726501465], "r_norm": [0.1327393501996994, 0.1310739368200302, 0.13362014293670654]}, {"step": 6800, "epoch": 14, "train_loss": 0.0010087769478559494, "cos_r_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_innovation_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_apical_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "cos_Ac_negg": [0.5887224674224854, 0.6323004364967346, 0.6633529663085938], "r_norm": [0.13503387570381165, 0.1333080381155014, 0.13559962809085846]}, {"step": 6900, "epoch": 14, "train_loss": 0.0009153951541520655, "cos_r_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_innovation_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_apical_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "cos_Ac_negg": [0.5893241763114929, 0.6298164129257202, 0.6629990935325623], "r_norm": [0.13242390751838684, 0.1308332085609436, 0.13339316844940186]}, {"step": 7000, "epoch": 14, "train_loss": 0.00029737711884081364, "cos_r_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_innovation_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_apical_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "cos_Ac_negg": [0.5876061916351318, 0.6274572610855103, 0.661839485168457], "r_norm": [0.1377774327993393, 0.1363024264574051, 0.13875912129878998], "ldr": {"dL_sdil": -0.007288441061973572, "dL_bp": -0.01291123777627945, "ratio": 0.564503666361401}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9742, "test_loss": 0.14128447875976563}], "final": {"test_acc": 0.9742, "test_loss": 0.14128447875976563, "wall_s": 29.368946313858032, "cos_r_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_innovation_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_apical_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591], "cos_Ac_negg": [0.5861812829971313, 0.6281393766403198, 0.6627663969993591]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_residual_s2.json b/results/nuis_ctrl_v1_rho0_residual_s2.json new file mode 100644 index 0000000..4c9dae4 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_residual_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_residual_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4856414794921875, "cos_r_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_innovation_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_apical_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "cos_Ac_negg": [0.027475744485855103, 0.03360234200954437, 0.0008582484442740679], "r_norm": [4.7544169425964355, 4.779862403869629, 4.861393451690674], "ldr": {"dL_sdil": -0.044302940368652344, "dL_bp": -0.24401354789733887, "ratio": 0.18155934680844618}}, {"step": 100, "epoch": 0, "train_loss": 0.3662866950035095, "cos_r_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_innovation_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_apical_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "cos_Ac_negg": [0.47842371463775635, 0.3934788703918457, 0.34560102224349976], "r_norm": [0.7926816940307617, 0.8032606840133667, 0.8174332976341248]}, {"step": 200, "epoch": 0, "train_loss": 0.2967141568660736, "cos_r_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_innovation_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_apical_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "cos_Ac_negg": [0.5089412927627563, 0.42410674691200256, 0.3532310724258423], "r_norm": [0.724601149559021, 0.7336956262588501, 0.7452678680419922]}, {"step": 300, "epoch": 0, "train_loss": 0.2878264784812927, "cos_r_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_innovation_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_apical_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "cos_Ac_negg": [0.5322810411453247, 0.4701629877090454, 0.3954809606075287], "r_norm": [0.6228486895561218, 0.6298609972000122, 0.6399674415588379]}, {"step": 400, "epoch": 0, "train_loss": 0.13541436195373535, "cos_r_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_innovation_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_apical_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "cos_Ac_negg": [0.5497274398803711, 0.4919236898422241, 0.42363661527633667], "r_norm": [0.5261503458023071, 0.5319482684135437, 0.5407342910766602]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9436, "test_loss": 0.18730589828491212}, {"step": 500, "epoch": 1, "train_loss": 0.2773134708404541, "cos_r_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_innovation_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_apical_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "cos_Ac_negg": [0.5579717755317688, 0.5103906393051147, 0.4453312158584595], "r_norm": [0.5363684892654419, 0.5415335893630981, 0.5485605597496033], "ldr": {"dL_sdil": -0.016582071781158447, "dL_bp": -0.013470649719238281, "ratio": 1.2309778761061947}}, {"step": 600, "epoch": 1, "train_loss": 0.24808740615844727, "cos_r_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_innovation_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_apical_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "cos_Ac_negg": [0.5471053123474121, 0.503724217414856, 0.4397326111793518], "r_norm": [0.4696118235588074, 0.4752689301967621, 0.48475584387779236]}, {"step": 700, "epoch": 1, "train_loss": 0.059591565281152725, "cos_r_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_innovation_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_apical_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "cos_Ac_negg": [0.5548915863037109, 0.5313657522201538, 0.4509887099266052], "r_norm": [0.4132225215435028, 0.417829692363739, 0.42398762702941895]}, {"step": 800, "epoch": 1, "train_loss": 0.08955289423465729, "cos_r_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_innovation_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_apical_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "cos_Ac_negg": [0.5715827941894531, 0.5440695285797119, 0.46786409616470337], "r_norm": [0.38012179732322693, 0.38466107845306396, 0.3904116451740265]}, {"step": 900, "epoch": 1, "train_loss": 0.12817199528217316, "cos_r_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_innovation_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_apical_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "cos_Ac_negg": [0.565535306930542, 0.5379549860954285, 0.4680260717868805], "r_norm": [0.42234253883361816, 0.42529693245887756, 0.43161889910697937]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9563, "test_loss": 0.14741420364379884}, {"step": 1000, "epoch": 2, "train_loss": 0.15121077001094818, "cos_r_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_innovation_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_apical_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "cos_Ac_negg": [0.5664821863174438, 0.5557536482810974, 0.5000541806221008], "r_norm": [0.38415396213531494, 0.38890281319618225, 0.3945630192756653], "ldr": {"dL_sdil": -0.01034557819366455, "dL_bp": -0.009327605366706848, "ratio": 1.109135494796035}}, {"step": 1100, "epoch": 2, "train_loss": 0.05521024763584137, "cos_r_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_innovation_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_apical_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "cos_Ac_negg": [0.5686019062995911, 0.5623975992202759, 0.495482861995697], "r_norm": [0.2886888384819031, 0.2918413281440735, 0.29741814732551575]}, {"step": 1200, "epoch": 2, "train_loss": 0.06215231865644455, "cos_r_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_innovation_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_apical_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "cos_Ac_negg": [0.5885026454925537, 0.576815664768219, 0.5003105401992798], "r_norm": [0.32788410782814026, 0.33229193091392517, 0.3380418121814728]}, {"step": 1300, "epoch": 2, "train_loss": 0.18930861353874207, "cos_r_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_innovation_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_apical_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "cos_Ac_negg": [0.5838833451271057, 0.5777159929275513, 0.5042821168899536], "r_norm": [0.33047181367874146, 0.334816038608551, 0.3400202989578247]}, {"step": 1400, "epoch": 2, "train_loss": 0.20265927910804749, "cos_r_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_innovation_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_apical_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "cos_Ac_negg": [0.5850098133087158, 0.582234263420105, 0.49856406450271606], "r_norm": [0.3220111131668091, 0.32452118396759033, 0.3309333324432373]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9623, "test_loss": 0.1362317768096924}, {"step": 1500, "epoch": 3, "train_loss": 0.054935283958911896, "cos_r_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_innovation_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_apical_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "cos_Ac_negg": [0.593195915222168, 0.5822055339813232, 0.5118049383163452], "r_norm": [0.29881566762924194, 0.30197426676750183, 0.3079264760017395], "ldr": {"dL_sdil": -0.008590251207351685, "dL_bp": -0.008395388722419739, "ratio": 1.0232106566324402}}, {"step": 1600, "epoch": 3, "train_loss": 0.044838763773441315, "cos_r_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_innovation_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_apical_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "cos_Ac_negg": [0.593943178653717, 0.6047523617744446, 0.5230796337127686], "r_norm": [0.26855069398880005, 0.2721232771873474, 0.27481386065483093]}, {"step": 1700, "epoch": 3, "train_loss": 0.08930075168609619, "cos_r_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_innovation_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_apical_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "cos_Ac_negg": [0.6031621098518372, 0.6102328300476074, 0.5353509187698364], "r_norm": [0.2840762436389923, 0.2851904630661011, 0.29301607608795166]}, {"step": 1800, "epoch": 3, "train_loss": 0.08093765377998352, "cos_r_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_innovation_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_apical_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "cos_Ac_negg": [0.5920141339302063, 0.600919246673584, 0.5265778303146362], "r_norm": [0.3048495054244995, 0.3076798915863037, 0.31106480956077576]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.968, "test_loss": 0.11007094802856446}, {"step": 1900, "epoch": 4, "train_loss": 0.04483484849333763, "cos_r_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_innovation_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_apical_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "cos_Ac_negg": [0.6020510792732239, 0.6103137135505676, 0.5334569215774536], "r_norm": [0.22422946989536285, 0.22653135657310486, 0.23203539848327637]}, {"step": 2000, "epoch": 4, "train_loss": 0.06511980295181274, "cos_r_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_innovation_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_apical_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "cos_Ac_negg": [0.6030604839324951, 0.6164731979370117, 0.5373966693878174], "r_norm": [0.22974814474582672, 0.2313867211341858, 0.23618879914283752], "ldr": {"dL_sdil": -0.005495421588420868, "dL_bp": -0.0060300081968307495, "ratio": 0.9113456249212315}}, {"step": 2100, "epoch": 4, "train_loss": 0.033010225743055344, "cos_r_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_innovation_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_apical_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "cos_Ac_negg": [0.5907387733459473, 0.6147665977478027, 0.540236234664917], "r_norm": [0.25794345140457153, 0.260486364364624, 0.26540467143058777]}, {"step": 2200, "epoch": 4, "train_loss": 0.10495644807815552, "cos_r_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_innovation_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_apical_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "cos_Ac_negg": [0.5863866209983826, 0.6134288311004639, 0.5443326830863953], "r_norm": [0.24134549498558044, 0.2427096962928772, 0.2470351755619049]}, {"step": 2300, "epoch": 4, "train_loss": 0.13655468821525574, "cos_r_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_innovation_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_apical_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "cos_Ac_negg": [0.586654543876648, 0.615277886390686, 0.5422926545143127], "r_norm": [0.2497454583644867, 0.2505592703819275, 0.25532519817352295]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.971, "test_loss": 0.1055252311706543}, {"step": 2400, "epoch": 5, "train_loss": 0.03528325632214546, "cos_r_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_innovation_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_apical_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "cos_Ac_negg": [0.5982511043548584, 0.6090170741081238, 0.5627652406692505], "r_norm": [0.2337566614151001, 0.23517361283302307, 0.24056139588356018]}, {"step": 2500, "epoch": 5, "train_loss": 0.051970984786748886, "cos_r_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_innovation_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_apical_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "cos_Ac_negg": [0.606495201587677, 0.6217761039733887, 0.5721772313117981], "r_norm": [0.20650900900363922, 0.20780494809150696, 0.21189066767692566], "ldr": {"dL_sdil": -0.0069409385323524475, "dL_bp": -0.00879254937171936, "ratio": 0.7894113798982473}}, {"step": 2600, "epoch": 5, "train_loss": 0.018195580691099167, "cos_r_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_innovation_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_apical_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "cos_Ac_negg": [0.5985366702079773, 0.6284464597702026, 0.5731967687606812], "r_norm": [0.19724297523498535, 0.1984039545059204, 0.2021041214466095]}, {"step": 2700, "epoch": 5, "train_loss": 0.0936257466673851, "cos_r_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_innovation_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_apical_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "cos_Ac_negg": [0.6147446036338806, 0.6406285166740417, 0.578402042388916], "r_norm": [0.20749273896217346, 0.209029883146286, 0.21408987045288086]}, {"step": 2800, "epoch": 5, "train_loss": 0.02398904599249363, "cos_r_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_innovation_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_apical_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "cos_Ac_negg": [0.6123343706130981, 0.6369959115982056, 0.573852002620697], "r_norm": [0.2375624030828476, 0.2401655614376068, 0.2447095513343811]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9716, "test_loss": 0.1127711856842041}, {"step": 2900, "epoch": 6, "train_loss": 0.02472660131752491, "cos_r_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_innovation_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_apical_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "cos_Ac_negg": [0.6068265438079834, 0.6319487690925598, 0.5736062526702881], "r_norm": [0.200751394033432, 0.20256194472312927, 0.20603583753108978]}, {"step": 3000, "epoch": 6, "train_loss": 0.01555752381682396, "cos_r_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_innovation_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_apical_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "cos_Ac_negg": [0.595432698726654, 0.6326295137405396, 0.5896763205528259], "r_norm": [0.22714081406593323, 0.22868841886520386, 0.2330799400806427], "ldr": {"dL_sdil": -0.008373714983463287, "dL_bp": -0.011146388947963715, "ratio": 0.7512491285344071}}, {"step": 3100, "epoch": 6, "train_loss": 0.012355584651231766, "cos_r_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_innovation_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_apical_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "cos_Ac_negg": [0.6013119220733643, 0.6345630288124084, 0.5907878279685974], "r_norm": [0.19545826315879822, 0.1969728022813797, 0.19908244907855988]}, {"step": 3200, "epoch": 6, "train_loss": 0.03212413936853409, "cos_r_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_innovation_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_apical_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "cos_Ac_negg": [0.5992377996444702, 0.6368730068206787, 0.6033105850219727], "r_norm": [0.20170357823371887, 0.20397330820560455, 0.20707929134368896]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9719, "test_loss": 0.11250891075134277}, {"step": 3300, "epoch": 7, "train_loss": 0.014137478545308113, "cos_r_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_innovation_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_apical_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "cos_Ac_negg": [0.6105959415435791, 0.6474651098251343, 0.6050795316696167], "r_norm": [0.19205130636692047, 0.19374202191829681, 0.1985575556755066]}, {"step": 3400, "epoch": 7, "train_loss": 0.015627946704626083, "cos_r_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_innovation_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_apical_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "cos_Ac_negg": [0.6015468835830688, 0.6357114315032959, 0.6087993383407593], "r_norm": [0.17368774116039276, 0.17372958362102509, 0.17921704053878784]}, {"step": 3500, "epoch": 7, "train_loss": 0.009410088881850243, "cos_r_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_innovation_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_apical_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "cos_Ac_negg": [0.6051868200302124, 0.6445883512496948, 0.6146015524864197], "r_norm": [0.15606719255447388, 0.1569913625717163, 0.16070200502872467], "ldr": {"dL_sdil": -0.005623310804367065, "dL_bp": -0.008359216153621674, "ratio": 0.6727079071798779}}, {"step": 3600, "epoch": 7, "train_loss": 0.007804212160408497, "cos_r_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_innovation_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_apical_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "cos_Ac_negg": [0.6069378852844238, 0.6444413661956787, 0.612177848815918], "r_norm": [0.1693708598613739, 0.16993819177150726, 0.17434334754943848]}, {"step": 3700, "epoch": 7, "train_loss": 0.002727900166064501, "cos_r_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_innovation_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_apical_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "cos_Ac_negg": [0.5997278690338135, 0.6434587836265564, 0.6153643131256104], "r_norm": [0.19530899822711945, 0.1965683102607727, 0.20004117488861084]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9735, "test_loss": 0.12146724281311035}, {"step": 3800, "epoch": 8, "train_loss": 0.0279134102165699, "cos_r_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_innovation_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_apical_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "cos_Ac_negg": [0.6049152612686157, 0.6492922306060791, 0.6219514608383179], "r_norm": [0.199128657579422, 0.1991393268108368, 0.20451344549655914]}, {"step": 3900, "epoch": 8, "train_loss": 0.007016683928668499, "cos_r_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_innovation_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_apical_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "cos_Ac_negg": [0.6019294261932373, 0.6473057866096497, 0.6265112161636353], "r_norm": [0.17034371197223663, 0.17111225426197052, 0.17593686282634735]}, {"step": 4000, "epoch": 8, "train_loss": 0.0011241799220442772, "cos_r_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_innovation_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_apical_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "cos_Ac_negg": [0.6019587516784668, 0.6427357792854309, 0.6214153170585632], "r_norm": [0.2055906504392624, 0.206048384308815, 0.2106199562549591], "ldr": {"dL_sdil": -0.009515531361103058, "dL_bp": -0.01432783156633377, "ratio": 0.6641292031560299}}, {"step": 4100, "epoch": 8, "train_loss": 0.052838996052742004, "cos_r_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_innovation_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_apical_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "cos_Ac_negg": [0.593733549118042, 0.6444112062454224, 0.6191511154174805], "r_norm": [0.18898266553878784, 0.1900315284729004, 0.19334247708320618]}, {"step": 4200, "epoch": 8, "train_loss": 0.010721303522586823, "cos_r_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_innovation_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_apical_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "cos_Ac_negg": [0.5992987155914307, 0.645348072052002, 0.6233259439468384], "r_norm": [0.1758223921060562, 0.17690947651863098, 0.1805993616580963]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9726, "test_loss": 0.12696008186340332}, {"step": 4300, "epoch": 9, "train_loss": 0.0010635580401867628, "cos_r_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_innovation_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_apical_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "cos_Ac_negg": [0.5912748575210571, 0.6386785507202148, 0.6203732490539551], "r_norm": [0.17538368701934814, 0.17650076746940613, 0.1804800033569336]}, {"step": 4400, "epoch": 9, "train_loss": 0.0015249300049617887, "cos_r_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_innovation_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_apical_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "cos_Ac_negg": [0.5900696516036987, 0.6396803855895996, 0.6211062669754028], "r_norm": [0.17721575498580933, 0.17846234142780304, 0.1818528175354004]}, {"step": 4500, "epoch": 9, "train_loss": 0.005601717159152031, "cos_r_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_innovation_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_apical_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "cos_Ac_negg": [0.5966706275939941, 0.6392136812210083, 0.6208544373512268], "r_norm": [0.16339799761772156, 0.16483958065509796, 0.16933569312095642], "ldr": {"dL_sdil": -0.0075685083866119385, "dL_bp": -0.012692451477050781, "ratio": 0.5962999661882936}}, {"step": 4600, "epoch": 9, "train_loss": 0.04552677646279335, "cos_r_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_innovation_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_apical_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "cos_Ac_negg": [0.5944240093231201, 0.6431910991668701, 0.6179784536361694], "r_norm": [0.16808894276618958, 0.16934075951576233, 0.1726376712322235]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9747, "test_loss": 0.12921665706634522}, {"step": 4700, "epoch": 10, "train_loss": 0.0016382120084017515, "cos_r_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_innovation_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_apical_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "cos_Ac_negg": [0.5882613658905029, 0.6381491422653198, 0.6210373640060425], "r_norm": [0.17644554376602173, 0.17762485146522522, 0.18118296563625336]}, {"step": 4800, "epoch": 10, "train_loss": 0.0031079151667654514, "cos_r_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_innovation_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_apical_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "cos_Ac_negg": [0.5947198867797852, 0.6479620933532715, 0.6301711797714233], "r_norm": [0.17002223432064056, 0.17095139622688293, 0.1749536097049713]}, {"step": 4900, "epoch": 10, "train_loss": 0.001899699796922505, "cos_r_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_innovation_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_apical_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "cos_Ac_negg": [0.5904074907302856, 0.6384178400039673, 0.6261191368103027], "r_norm": [0.1906973421573639, 0.19160214066505432, 0.19531694054603577]}, {"step": 5000, "epoch": 10, "train_loss": 0.002346828579902649, "cos_r_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_innovation_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_apical_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "cos_Ac_negg": [0.5909355878829956, 0.6369606852531433, 0.6278571486473083], "r_norm": [0.17863205075263977, 0.18014976382255554, 0.18399974703788757], "ldr": {"dL_sdil": -0.008974842727184296, "dL_bp": -0.015554085373878479, "ratio": 0.5770087093810505}}, {"step": 5100, "epoch": 10, "train_loss": 0.0018325410783290863, "cos_r_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_innovation_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_apical_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "cos_Ac_negg": [0.5919134616851807, 0.6349148750305176, 0.6300892233848572], "r_norm": [0.17877718806266785, 0.1797236204147339, 0.183450847864151]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9742, "test_loss": 0.1357520221710205}, {"step": 5200, "epoch": 11, "train_loss": 0.0008780563366599381, "cos_r_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_innovation_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_apical_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "cos_Ac_negg": [0.5842093825340271, 0.6340514421463013, 0.6265090107917786], "r_norm": [0.17967163026332855, 0.18078157305717468, 0.1842673122882843]}, {"step": 5300, "epoch": 11, "train_loss": 0.0013086458202451468, "cos_r_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_innovation_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_apical_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "cos_Ac_negg": [0.5804507732391357, 0.6330962777137756, 0.6264883279800415], "r_norm": [0.17678910493850708, 0.1777363121509552, 0.18114155530929565]}, {"step": 5400, "epoch": 11, "train_loss": 0.0006866963813081384, "cos_r_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_innovation_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_apical_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "cos_Ac_negg": [0.5720582008361816, 0.631247341632843, 0.6274412274360657], "r_norm": [0.18510422110557556, 0.18618690967559814, 0.1894882768392563]}, {"step": 5500, "epoch": 11, "train_loss": 0.0008870868477970362, "cos_r_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_innovation_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_apical_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "cos_Ac_negg": [0.5726954340934753, 0.6326087117195129, 0.629157543182373], "r_norm": [0.17689445614814758, 0.17830178141593933, 0.18123194575309753], "ldr": {"dL_sdil": -0.008988067507743835, "dL_bp": -0.01625603437423706, "ratio": 0.5529065269441318}}, {"step": 5600, "epoch": 11, "train_loss": 0.0004585091082844883, "cos_r_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_innovation_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_apical_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "cos_Ac_negg": [0.5735850930213928, 0.6342295408248901, 0.6308751702308655], "r_norm": [0.17601367831230164, 0.17745786905288696, 0.18061557412147522]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9754, "test_loss": 0.13874979991912842}, {"step": 5700, "epoch": 12, "train_loss": 0.0007273177616298199, "cos_r_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_innovation_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_apical_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "cos_Ac_negg": [0.5749819278717041, 0.6299868226051331, 0.627753734588623], "r_norm": [0.1876221001148224, 0.18882854282855988, 0.1921488344669342]}, {"step": 5800, "epoch": 12, "train_loss": 0.00034042136394418776, "cos_r_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_innovation_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_apical_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "cos_Ac_negg": [0.5690767765045166, 0.6268764734268188, 0.628614068031311], "r_norm": [0.1746070384979248, 0.17565712332725525, 0.17919966578483582]}, {"step": 5900, "epoch": 12, "train_loss": 0.0014644403709098697, "cos_r_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_innovation_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_apical_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "cos_Ac_negg": [0.5693458914756775, 0.6279462575912476, 0.6293314695358276], "r_norm": [0.171538308262825, 0.17262454330921173, 0.176008403301239]}, {"step": 6000, "epoch": 12, "train_loss": 0.0009379165712743998, "cos_r_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_innovation_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_apical_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "cos_Ac_negg": [0.5666444897651672, 0.6272848844528198, 0.6319928169250488], "r_norm": [0.17355795204639435, 0.17495247721672058, 0.17821210622787476], "ldr": {"dL_sdil": -0.008706547319889069, "dL_bp": -0.015992946922779083, "ratio": 0.5443991880876035}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9754, "test_loss": 0.14179479808807374}, {"step": 6100, "epoch": 13, "train_loss": 0.001136131351813674, "cos_r_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_innovation_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_apical_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "cos_Ac_negg": [0.5693786144256592, 0.6286900043487549, 0.6344629526138306], "r_norm": [0.17050513625144958, 0.1718040406703949, 0.17503532767295837]}, {"step": 6200, "epoch": 13, "train_loss": 0.00029565332806669176, "cos_r_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_innovation_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_apical_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "cos_Ac_negg": [0.5668879747390747, 0.6276275515556335, 0.6326006054878235], "r_norm": [0.1782611608505249, 0.17942413687705994, 0.18300361931324005]}, {"step": 6300, "epoch": 13, "train_loss": 0.00017874882905744016, "cos_r_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_innovation_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_apical_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "cos_Ac_negg": [0.5669001936912537, 0.627455472946167, 0.6325979232788086], "r_norm": [0.1737596094608307, 0.17501607537269592, 0.1786150336265564]}, {"step": 6400, "epoch": 13, "train_loss": 0.0014268780359998345, "cos_r_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_innovation_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_apical_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "cos_Ac_negg": [0.5664763450622559, 0.6264446973800659, 0.6344162225723267], "r_norm": [0.17299015820026398, 0.17426249384880066, 0.1777229905128479]}, {"step": 6500, "epoch": 13, "train_loss": 0.00022807065397500992, "cos_r_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_innovation_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_apical_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "cos_Ac_negg": [0.5625617504119873, 0.6261914968490601, 0.6329864263534546], "r_norm": [0.1683826446533203, 0.16965822875499725, 0.17293044924736023], "ldr": {"dL_sdil": -0.008848324418067932, "dL_bp": -0.017275981605052948, "ratio": 0.5121749154606612}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9744, "test_loss": 0.14613630256652832}, {"step": 6600, "epoch": 14, "train_loss": 0.0008223638869822025, "cos_r_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_innovation_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_apical_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "cos_Ac_negg": [0.5643479824066162, 0.6252572536468506, 0.6311848163604736], "r_norm": [0.17072054743766785, 0.17188414931297302, 0.17549914121627808]}, {"step": 6700, "epoch": 14, "train_loss": 0.0012336993822827935, "cos_r_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_innovation_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_apical_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "cos_Ac_negg": [0.563581109046936, 0.6277950406074524, 0.6328113079071045], "r_norm": [0.17281609773635864, 0.17397551238536835, 0.1775282919406891]}, {"step": 6800, "epoch": 14, "train_loss": 0.0017774601001292467, "cos_r_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_innovation_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_apical_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "cos_Ac_negg": [0.5627243518829346, 0.6263659596443176, 0.6334640979766846], "r_norm": [0.1771830916404724, 0.17863672971725464, 0.18169184029102325]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006432391237467527, "cos_r_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_innovation_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_apical_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "cos_Ac_negg": [0.5642813444137573, 0.6263474225997925, 0.634334921836853], "r_norm": [0.1692524403333664, 0.17056486010551453, 0.17403511703014374]}, {"step": 7000, "epoch": 14, "train_loss": 0.000249928911216557, "cos_r_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_innovation_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_apical_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "cos_Ac_negg": [0.5637162923812866, 0.6274820566177368, 0.6343455910682678], "r_norm": [0.1676335632801056, 0.16881629824638367, 0.17234541475772858], "ldr": {"dL_sdil": -0.008975468575954437, "dL_bp": -0.017749518156051636, "ratio": 0.505673928556437}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9742, "test_loss": 0.14835414848327635}], "final": {"test_acc": 0.9742, "test_loss": 0.14835414848327635, "wall_s": 29.85836362838745, "cos_r_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_innovation_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_apical_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905], "cos_Ac_negg": [0.5650476813316345, 0.6291353702545166, 0.635208785533905]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_residual_s3.json b/results/nuis_ctrl_v1_rho0_residual_s3.json new file mode 100644 index 0000000..c3b1bdf --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_residual_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_residual_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3283889293670654, "cos_r_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_innovation_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_apical_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "cos_Ac_negg": [-0.008537412621080875, 0.008568145334720612, 0.004687415435910225], "r_norm": [4.841709613800049, 4.737439155578613, 4.835896015167236], "ldr": {"dL_sdil": -0.0037169456481933594, "dL_bp": -0.22416257858276367, "ratio": 0.016581472570904674}}, {"step": 100, "epoch": 0, "train_loss": 0.33480948209762573, "cos_r_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_innovation_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_apical_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "cos_Ac_negg": [0.42985573410987854, 0.3650231957435608, 0.35172039270401], "r_norm": [0.7733307480812073, 0.758333683013916, 0.7694300413131714]}, {"step": 200, "epoch": 0, "train_loss": 0.32067805528640747, "cos_r_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_innovation_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_apical_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "cos_Ac_negg": [0.4714224636554718, 0.4232518970966339, 0.39167487621307373], "r_norm": [0.7619251608848572, 0.7530097961425781, 0.7603987455368042]}, {"step": 300, "epoch": 0, "train_loss": 0.3700808882713318, "cos_r_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_innovation_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_apical_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "cos_Ac_negg": [0.5015236139297485, 0.45720142126083374, 0.4300490617752075], "r_norm": [0.5746974945068359, 0.5715751647949219, 0.5759649276733398]}, {"step": 400, "epoch": 0, "train_loss": 0.11992092430591583, "cos_r_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_innovation_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_apical_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "cos_Ac_negg": [0.5079100131988525, 0.47483697533607483, 0.4512515068054199], "r_norm": [0.4775196313858032, 0.47117429971694946, 0.4762042164802551]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9485, "test_loss": 0.17359396591186524}, {"step": 500, "epoch": 1, "train_loss": 0.25980252027511597, "cos_r_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_innovation_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_apical_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "cos_Ac_negg": [0.5498027801513672, 0.5152735710144043, 0.46535003185272217], "r_norm": [0.5102050304412842, 0.5090956091880798, 0.5102525949478149], "ldr": {"dL_sdil": -0.014272913336753845, "dL_bp": -0.011847808957099915, "ratio": 1.2046880050509814}}, {"step": 600, "epoch": 1, "train_loss": 0.20059989392757416, "cos_r_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_innovation_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_apical_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "cos_Ac_negg": [0.5179136991500854, 0.4873511493206024, 0.45999011397361755], "r_norm": [0.4354090094566345, 0.4269413948059082, 0.4325163960456848]}, {"step": 700, "epoch": 1, "train_loss": 0.08970798552036285, "cos_r_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_innovation_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_apical_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "cos_Ac_negg": [0.5299416780471802, 0.5087264776229858, 0.47246772050857544], "r_norm": [0.4034784436225891, 0.4017947316169739, 0.4032314419746399]}, {"step": 800, "epoch": 1, "train_loss": 0.10161302983760834, "cos_r_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_innovation_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_apical_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "cos_Ac_negg": [0.5301811695098877, 0.4997856616973877, 0.46729350090026855], "r_norm": [0.4015730917453766, 0.39593786001205444, 0.4005199074745178]}, {"step": 900, "epoch": 1, "train_loss": 0.12729158997535706, "cos_r_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_innovation_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_apical_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "cos_Ac_negg": [0.5222463011741638, 0.5066336393356323, 0.48333653807640076], "r_norm": [0.3631213307380676, 0.3588687479496002, 0.36084190011024475]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9597, "test_loss": 0.1398345920562744}, {"step": 1000, "epoch": 2, "train_loss": 0.13055379688739777, "cos_r_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_innovation_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_apical_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "cos_Ac_negg": [0.5575971007347107, 0.5522615909576416, 0.49721530079841614], "r_norm": [0.3458414077758789, 0.339888334274292, 0.3438340425491333], "ldr": {"dL_sdil": -0.00751737505197525, "dL_bp": -0.006968997418880463, "ratio": 1.0786881670538604}}, {"step": 1100, "epoch": 2, "train_loss": 0.08165145665407181, "cos_r_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_innovation_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_apical_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "cos_Ac_negg": [0.5387380123138428, 0.5444086194038391, 0.4962887763977051], "r_norm": [0.27777719497680664, 0.27354109287261963, 0.2770301103591919]}, {"step": 1200, "epoch": 2, "train_loss": 0.03162624314427376, "cos_r_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_innovation_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_apical_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "cos_Ac_negg": [0.5260978937149048, 0.5270414352416992, 0.4954911172389984], "r_norm": [0.3397935926914215, 0.3343070149421692, 0.33784517645835876]}, {"step": 1300, "epoch": 2, "train_loss": 0.12580201029777527, "cos_r_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_innovation_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_apical_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "cos_Ac_negg": [0.5428162813186646, 0.5567272305488586, 0.5213391780853271], "r_norm": [0.27239900827407837, 0.26863694190979004, 0.2723345160484314]}, {"step": 1400, "epoch": 2, "train_loss": 0.20885255932807922, "cos_r_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_innovation_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_apical_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "cos_Ac_negg": [0.5399730205535889, 0.5484820604324341, 0.5143886804580688], "r_norm": [0.28959929943084717, 0.28440648317337036, 0.2864704132080078]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9624, "test_loss": 0.127296443939209}, {"step": 1500, "epoch": 3, "train_loss": 0.07511237263679504, "cos_r_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_innovation_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_apical_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "cos_Ac_negg": [0.5317904949188232, 0.5460232496261597, 0.5112682580947876], "r_norm": [0.31371599435806274, 0.30785977840423584, 0.31208980083465576], "ldr": {"dL_sdil": -0.007627889513969421, "dL_bp": -0.006990775465965271, "ratio": 1.0911363912495764}}, {"step": 1600, "epoch": 3, "train_loss": 0.05954476445913315, "cos_r_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_innovation_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_apical_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "cos_Ac_negg": [0.5486156940460205, 0.5706946849822998, 0.5293116569519043], "r_norm": [0.2844116687774658, 0.28088948130607605, 0.2819110155105591]}, {"step": 1700, "epoch": 3, "train_loss": 0.05954836308956146, "cos_r_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_innovation_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_apical_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "cos_Ac_negg": [0.5640909671783447, 0.5758436918258667, 0.5190476179122925], "r_norm": [0.27005547285079956, 0.2676847577095032, 0.2688477635383606]}, {"step": 1800, "epoch": 3, "train_loss": 0.04903425648808479, "cos_r_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_innovation_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_apical_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "cos_Ac_negg": [0.549634575843811, 0.5774312019348145, 0.5283776521682739], "r_norm": [0.2799452543258667, 0.2738724946975708, 0.27548953890800476]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9699, "test_loss": 0.10895987529754639}, {"step": 1900, "epoch": 4, "train_loss": 0.05801629275083542, "cos_r_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_innovation_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_apical_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "cos_Ac_negg": [0.5774978399276733, 0.587746262550354, 0.5384795069694519], "r_norm": [0.22653728723526, 0.22427527606487274, 0.2236478626728058]}, {"step": 2000, "epoch": 4, "train_loss": 0.10690974444150925, "cos_r_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_innovation_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_apical_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "cos_Ac_negg": [0.5727092027664185, 0.583440899848938, 0.5408304333686829], "r_norm": [0.2192738950252533, 0.21646450459957123, 0.21635594964027405], "ldr": {"dL_sdil": -0.004961423575878143, "dL_bp": -0.005419991910457611, "ratio": 0.9153931699243532}}, {"step": 2100, "epoch": 4, "train_loss": 0.04904601722955704, "cos_r_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_innovation_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_apical_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "cos_Ac_negg": [0.5653552412986755, 0.5845832824707031, 0.5582318305969238], "r_norm": [0.25469183921813965, 0.24852119386196136, 0.2508085072040558]}, {"step": 2200, "epoch": 4, "train_loss": 0.07027833163738251, "cos_r_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_innovation_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_apical_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "cos_Ac_negg": [0.5711768269538879, 0.5911603569984436, 0.5652821063995361], "r_norm": [0.22217589616775513, 0.21898141503334045, 0.2202150672674179]}, {"step": 2300, "epoch": 4, "train_loss": 0.1405230015516281, "cos_r_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_innovation_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_apical_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "cos_Ac_negg": [0.5626112222671509, 0.5882081985473633, 0.5767524838447571], "r_norm": [0.2517160177230835, 0.24741223454475403, 0.2489166259765625]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9701, "test_loss": 0.10518000926971435}, {"step": 2400, "epoch": 5, "train_loss": 0.015669789165258408, "cos_r_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_innovation_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_apical_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "cos_Ac_negg": [0.5766092538833618, 0.5981851816177368, 0.5789803266525269], "r_norm": [0.2449054718017578, 0.24151459336280823, 0.24172821640968323]}, {"step": 2500, "epoch": 5, "train_loss": 0.04790015518665314, "cos_r_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_innovation_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_apical_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "cos_Ac_negg": [0.5918060541152954, 0.6062850952148438, 0.5871388912200928], "r_norm": [0.2145114541053772, 0.21184881031513214, 0.21148696541786194], "ldr": {"dL_sdil": -0.005982436239719391, "dL_bp": -0.0073370859026908875, "ratio": 0.8153695239584592}}, {"step": 2600, "epoch": 5, "train_loss": 0.014841018244624138, "cos_r_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_innovation_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_apical_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "cos_Ac_negg": [0.5842347145080566, 0.6029914021492004, 0.5868893265724182], "r_norm": [0.19980891048908234, 0.19781577587127686, 0.19805651903152466]}, {"step": 2700, "epoch": 5, "train_loss": 0.07935170829296112, "cos_r_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_innovation_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_apical_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "cos_Ac_negg": [0.5844300985336304, 0.608223557472229, 0.594878077507019], "r_norm": [0.23918147385120392, 0.23707768321037292, 0.2370166778564453]}, {"step": 2800, "epoch": 5, "train_loss": 0.005857329349964857, "cos_r_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_innovation_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_apical_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "cos_Ac_negg": [0.5927610397338867, 0.6110668778419495, 0.5913129448890686], "r_norm": [0.21699143946170807, 0.2136717587709427, 0.21461525559425354]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.971, "test_loss": 0.10945739488601684}, {"step": 2900, "epoch": 6, "train_loss": 0.0123747568577528, "cos_r_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_innovation_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_apical_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "cos_Ac_negg": [0.5904170274734497, 0.616287112236023, 0.6065865755081177], "r_norm": [0.23866206407546997, 0.23430858552455902, 0.23588283360004425]}, {"step": 3000, "epoch": 6, "train_loss": 0.008246966637670994, "cos_r_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_innovation_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_apical_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "cos_Ac_negg": [0.5872467160224915, 0.6118901371955872, 0.6088818311691284], "r_norm": [0.24846184253692627, 0.2451106160879135, 0.24654726684093475], "ldr": {"dL_sdil": -0.008115455508232117, "dL_bp": -0.010389789938926697, "ratio": 0.7810990940082926}}, {"step": 3100, "epoch": 6, "train_loss": 0.011437802575528622, "cos_r_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_innovation_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_apical_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "cos_Ac_negg": [0.5957895517349243, 0.613793671131134, 0.6057416200637817], "r_norm": [0.19606490433216095, 0.1932457983493805, 0.19307956099510193]}, {"step": 3200, "epoch": 6, "train_loss": 0.032173387706279755, "cos_r_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_innovation_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_apical_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "cos_Ac_negg": [0.5974835157394409, 0.6127585768699646, 0.6073087453842163], "r_norm": [0.2475024312734604, 0.24428103864192963, 0.24396109580993652]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.971, "test_loss": 0.11738142681121826}, {"step": 3300, "epoch": 7, "train_loss": 0.0035855344031006098, "cos_r_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_innovation_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_apical_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "cos_Ac_negg": [0.5997357964515686, 0.6208878755569458, 0.6059156656265259], "r_norm": [0.1960332989692688, 0.1931273192167282, 0.19294635951519012]}, {"step": 3400, "epoch": 7, "train_loss": 0.004453396890312433, "cos_r_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_innovation_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_apical_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "cos_Ac_negg": [0.6008877754211426, 0.6181595325469971, 0.6077213883399963], "r_norm": [0.22042231261730194, 0.21685011684894562, 0.21672581136226654]}, {"step": 3500, "epoch": 7, "train_loss": 0.00869251973927021, "cos_r_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_innovation_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_apical_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "cos_Ac_negg": [0.6080660223960876, 0.6224737167358398, 0.6180291771888733], "r_norm": [0.21137651801109314, 0.2083500772714615, 0.20858825743198395], "ldr": {"dL_sdil": -0.006606429815292358, "dL_bp": -0.009430110454559326, "ratio": 0.7005675964376686}}, {"step": 3600, "epoch": 7, "train_loss": 0.004775376059114933, "cos_r_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_innovation_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_apical_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "cos_Ac_negg": [0.6005524396896362, 0.6210868954658508, 0.6165405511856079], "r_norm": [0.2073669135570526, 0.20420829951763153, 0.20337089896202087]}, {"step": 3700, "epoch": 7, "train_loss": 0.0054521383717656136, "cos_r_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_innovation_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_apical_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "cos_Ac_negg": [0.6015474200248718, 0.6185621023178101, 0.6208216547966003], "r_norm": [0.20405349135398865, 0.20031914114952087, 0.20035898685455322]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9734, "test_loss": 0.11372013940811157}, {"step": 3800, "epoch": 8, "train_loss": 0.015979381278157234, "cos_r_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_innovation_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_apical_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "cos_Ac_negg": [0.5952907800674438, 0.624449610710144, 0.6298924088478088], "r_norm": [0.19954121112823486, 0.19605880975723267, 0.1963454782962799]}, {"step": 3900, "epoch": 8, "train_loss": 0.017817214131355286, "cos_r_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_innovation_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_apical_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "cos_Ac_negg": [0.5973081588745117, 0.6232448816299438, 0.6271523237228394], "r_norm": [0.20743486285209656, 0.20354314148426056, 0.20332831144332886]}, {"step": 4000, "epoch": 8, "train_loss": 0.0004270627978257835, "cos_r_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_innovation_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_apical_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "cos_Ac_negg": [0.6016135811805725, 0.6266351342201233, 0.6324046850204468], "r_norm": [0.19953644275665283, 0.19602517783641815, 0.1954791247844696], "ldr": {"dL_sdil": -0.008086413145065308, "dL_bp": -0.013367392122745514, "ratio": 0.6049357324758756}}, {"step": 4100, "epoch": 8, "train_loss": 0.07452463358640671, "cos_r_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_innovation_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_apical_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "cos_Ac_negg": [0.6088876724243164, 0.6224457025527954, 0.636536717414856], "r_norm": [0.20760531723499298, 0.20430520176887512, 0.20437899231910706]}, {"step": 4200, "epoch": 8, "train_loss": 0.03605803847312927, "cos_r_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_innovation_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_apical_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "cos_Ac_negg": [0.6000900268554688, 0.6237789392471313, 0.6348117589950562], "r_norm": [0.19332198798656464, 0.1901654452085495, 0.18981432914733887]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9724, "test_loss": 0.11489360466003418}, {"step": 4300, "epoch": 9, "train_loss": 0.0038651630748063326, "cos_r_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_innovation_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_apical_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "cos_Ac_negg": [0.6032973527908325, 0.6257011890411377, 0.6360322833061218], "r_norm": [0.21041682362556458, 0.20728406310081482, 0.20620422065258026]}, {"step": 4400, "epoch": 9, "train_loss": 0.0014019431546330452, "cos_r_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_innovation_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_apical_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "cos_Ac_negg": [0.6007627248764038, 0.6224915981292725, 0.6380206346511841], "r_norm": [0.19830866158008575, 0.19524846971035004, 0.19488275051116943]}, {"step": 4500, "epoch": 9, "train_loss": 0.0025219277013093233, "cos_r_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_innovation_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_apical_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "cos_Ac_negg": [0.5988080501556396, 0.6245213747024536, 0.6350066661834717], "r_norm": [0.19933810830116272, 0.195916086435318, 0.19609415531158447], "ldr": {"dL_sdil": -0.008369803428649902, "dL_bp": -0.014350831508636475, "ratio": 0.5832277679250063}}, {"step": 4600, "epoch": 9, "train_loss": 0.06718802452087402, "cos_r_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_innovation_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_apical_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "cos_Ac_negg": [0.5992623567581177, 0.6246331334114075, 0.6348356604576111], "r_norm": [0.19856536388397217, 0.19568754732608795, 0.19562330842018127]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9747, "test_loss": 0.11986219282150269}, {"step": 4700, "epoch": 10, "train_loss": 0.0012022472219541669, "cos_r_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_innovation_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_apical_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "cos_Ac_negg": [0.5960962772369385, 0.619342029094696, 0.6362457871437073], "r_norm": [0.19958841800689697, 0.19681866466999054, 0.19656816124916077]}, {"step": 4800, "epoch": 10, "train_loss": 0.0013852894771844149, "cos_r_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_innovation_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_apical_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "cos_Ac_negg": [0.6009965538978577, 0.6231017112731934, 0.642189621925354], "r_norm": [0.19555793702602386, 0.1924840807914734, 0.1920878291130066]}, {"step": 4900, "epoch": 10, "train_loss": 0.0019364049658179283, "cos_r_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_innovation_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_apical_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "cos_Ac_negg": [0.6010047197341919, 0.6248171329498291, 0.6422927975654602], "r_norm": [0.19706960022449493, 0.1942807137966156, 0.19380313158035278]}, {"step": 5000, "epoch": 10, "train_loss": 0.0011562240542843938, "cos_r_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_innovation_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_apical_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "cos_Ac_negg": [0.5932005643844604, 0.6226108074188232, 0.6409620046615601], "r_norm": [0.19638454914093018, 0.19321104884147644, 0.1927313357591629], "ldr": {"dL_sdil": -0.008506037294864655, "dL_bp": -0.015625961124897003, "ratio": 0.544352902639179}}, {"step": 5100, "epoch": 10, "train_loss": 0.0011195843107998371, "cos_r_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_innovation_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_apical_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "cos_Ac_negg": [0.5915175676345825, 0.6213323473930359, 0.6448142528533936], "r_norm": [0.21668797731399536, 0.21310555934906006, 0.21348032355308533]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9736, "test_loss": 0.12488943996429443}, {"step": 5200, "epoch": 11, "train_loss": 0.0005131271318532526, "cos_r_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_innovation_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_apical_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "cos_Ac_negg": [0.5931934118270874, 0.6207455396652222, 0.6424674987792969], "r_norm": [0.2030562162399292, 0.1994873583316803, 0.19953997433185577]}, {"step": 5300, "epoch": 11, "train_loss": 0.0014590469654649496, "cos_r_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_innovation_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_apical_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "cos_Ac_negg": [0.5942462682723999, 0.6223618984222412, 0.6452451348304749], "r_norm": [0.20533114671707153, 0.20210321247577667, 0.20187830924987793]}, {"step": 5400, "epoch": 11, "train_loss": 0.0009648285340517759, "cos_r_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_innovation_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_apical_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "cos_Ac_negg": [0.591279149055481, 0.6222770810127258, 0.644339382648468], "r_norm": [0.2028876543045044, 0.19992581009864807, 0.19973811507225037]}, {"step": 5500, "epoch": 11, "train_loss": 0.002808267716318369, "cos_r_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_innovation_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_apical_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "cos_Ac_negg": [0.5916488170623779, 0.6226444244384766, 0.6463583707809448], "r_norm": [0.19786036014556885, 0.19502316415309906, 0.19484545290470123], "ldr": {"dL_sdil": -0.009292647242546082, "dL_bp": -0.017724603414535522, "ratio": 0.5242795579237279}}, {"step": 5600, "epoch": 11, "train_loss": 0.0007443465292453766, "cos_r_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_innovation_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_apical_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "cos_Ac_negg": [0.5919091105461121, 0.6228354573249817, 0.6468508839607239], "r_norm": [0.19685988128185272, 0.1940140724182129, 0.19382789731025696]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9741, "test_loss": 0.12913100547790526}, {"step": 5700, "epoch": 12, "train_loss": 0.00026087171863764524, "cos_r_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_innovation_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_apical_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "cos_Ac_negg": [0.5873425006866455, 0.6209858059883118, 0.645824670791626], "r_norm": [0.20257383584976196, 0.19969847798347473, 0.19937360286712646]}, {"step": 5800, "epoch": 12, "train_loss": 0.0011157530825585127, "cos_r_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_innovation_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_apical_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "cos_Ac_negg": [0.5874757766723633, 0.6192538142204285, 0.6464934945106506], "r_norm": [0.20035849511623383, 0.1976122111082077, 0.1971856951713562]}, {"step": 5900, "epoch": 12, "train_loss": 0.0022946004755795, "cos_r_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_innovation_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_apical_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "cos_Ac_negg": [0.5894894599914551, 0.6217609643936157, 0.6475809216499329], "r_norm": [0.20160531997680664, 0.19882169365882874, 0.1985234022140503]}, {"step": 6000, "epoch": 12, "train_loss": 0.0015348243759945035, "cos_r_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_innovation_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_apical_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "cos_Ac_negg": [0.5849478244781494, 0.6191039681434631, 0.6462820172309875], "r_norm": [0.21117167174816132, 0.208292156457901, 0.20826399326324463], "ldr": {"dL_sdil": -0.010774381458759308, "dL_bp": -0.021835722029209137, "ratio": 0.49342913618091805}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9747, "test_loss": 0.13031967477798462}, {"step": 6100, "epoch": 13, "train_loss": 0.0005886715371161699, "cos_r_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_innovation_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_apical_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "cos_Ac_negg": [0.5848478078842163, 0.6192096471786499, 0.6462153196334839], "r_norm": [0.20275822281837463, 0.1999770700931549, 0.1997852921485901]}, {"step": 6200, "epoch": 13, "train_loss": 0.00026136147789657116, "cos_r_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_innovation_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_apical_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "cos_Ac_negg": [0.5864197015762329, 0.617784857749939, 0.6452604532241821], "r_norm": [0.19926416873931885, 0.19659274816513062, 0.1963043063879013]}, {"step": 6300, "epoch": 13, "train_loss": 0.0008323256624862552, "cos_r_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_innovation_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_apical_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "cos_Ac_negg": [0.5841127038002014, 0.6188559532165527, 0.6470884680747986], "r_norm": [0.19856512546539307, 0.19585520029067993, 0.19567559659481049]}, {"step": 6400, "epoch": 13, "train_loss": 0.0005636445712298155, "cos_r_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_innovation_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_apical_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "cos_Ac_negg": [0.584000289440155, 0.6188395619392395, 0.6467247009277344], "r_norm": [0.2014485001564026, 0.19853657484054565, 0.19835329055786133]}, {"step": 6500, "epoch": 13, "train_loss": 0.00021077357814647257, "cos_r_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_innovation_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_apical_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "cos_Ac_negg": [0.5866034626960754, 0.6200528144836426, 0.6460997462272644], "r_norm": [0.19778192043304443, 0.19491931796073914, 0.19479045271873474], "ldr": {"dL_sdil": -0.010284215211868286, "dL_bp": -0.021059155464172363, "ratio": 0.4883488907883639}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9752, "test_loss": 0.13342348890304565}, {"step": 6600, "epoch": 14, "train_loss": 0.0004418176249600947, "cos_r_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_innovation_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_apical_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "cos_Ac_negg": [0.5835412740707397, 0.6186872720718384, 0.6473947763442993], "r_norm": [0.20090188086032867, 0.19807308912277222, 0.19789955019950867]}, {"step": 6700, "epoch": 14, "train_loss": 0.001412956160493195, "cos_r_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_innovation_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_apical_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "cos_Ac_negg": [0.5821904540061951, 0.6165207028388977, 0.6474819779396057], "r_norm": [0.2016269862651825, 0.19889488816261292, 0.19875365495681763]}, {"step": 6800, "epoch": 14, "train_loss": 0.0006913552642799914, "cos_r_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_innovation_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_apical_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "cos_Ac_negg": [0.5829519033432007, 0.6182739734649658, 0.6466283798217773], "r_norm": [0.20099763572216034, 0.198038712143898, 0.19800257682800293]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006930493982508779, "cos_r_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_innovation_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_apical_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "cos_Ac_negg": [0.5831266641616821, 0.6174263954162598, 0.6491556167602539], "r_norm": [0.19700448215007782, 0.19417372345924377, 0.19408582150936127]}, {"step": 7000, "epoch": 14, "train_loss": 0.00032816026941873133, "cos_r_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_innovation_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_apical_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "cos_Ac_negg": [0.5834798812866211, 0.6168712973594666, 0.6480824947357178], "r_norm": [0.19693708419799805, 0.19410592317581177, 0.19396330416202545], "ldr": {"dL_sdil": -0.010280802845954895, "dL_bp": -0.021565861999988556, "ratio": 0.47671652753598953}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9753, "test_loss": 0.1358968108177185}], "final": {"test_acc": 0.9753, "test_loss": 0.1358968108177185, "wall_s": 21.215047121047974, "cos_r_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_innovation_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_apical_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076], "cos_Ac_negg": [0.5830169916152954, 0.6173279285430908, 0.6489131450653076]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0_residual_s4.json b/results/nuis_ctrl_v1_rho0_residual_s4.json new file mode 100644 index 0000000..e568b27 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0_residual_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0_residual_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4173901081085205, "cos_r_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_innovation_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_apical_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "cos_Ac_negg": [-0.013576557859778404, 0.025841742753982544, 0.0074026198126375675], "r_norm": [4.844976425170898, 4.853462219238281, 4.948795318603516], "ldr": {"dL_sdil": -0.00283050537109375, "dL_bp": -0.25957250595092773, "ratio": 0.01090448836530036}}, {"step": 100, "epoch": 0, "train_loss": 0.4020535349845886, "cos_r_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_innovation_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_apical_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "cos_Ac_negg": [0.47780513763427734, 0.3931659460067749, 0.32962289452552795], "r_norm": [0.7995222806930542, 0.8062721490859985, 0.8229418396949768]}, {"step": 200, "epoch": 0, "train_loss": 0.2508821189403534, "cos_r_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_innovation_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_apical_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "cos_Ac_negg": [0.497688353061676, 0.4222521185874939, 0.358740895986557], "r_norm": [0.7765392065048218, 0.7898226380348206, 0.8046278357505798]}, {"step": 300, "epoch": 0, "train_loss": 0.25044649839401245, "cos_r_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_innovation_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_apical_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "cos_Ac_negg": [0.5308647155761719, 0.454055517911911, 0.3798382878303528], "r_norm": [0.6173285841941833, 0.623796284198761, 0.6384584903717041]}, {"step": 400, "epoch": 0, "train_loss": 0.17270351946353912, "cos_r_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_innovation_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_apical_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "cos_Ac_negg": [0.5254049301147461, 0.4671183228492737, 0.39171653985977173], "r_norm": [0.5093363523483276, 0.5141817331314087, 0.5286765098571777]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9455, "test_loss": 0.1889075668334961}, {"step": 500, "epoch": 1, "train_loss": 0.21437089145183563, "cos_r_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_innovation_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_apical_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "cos_Ac_negg": [0.576738178730011, 0.512541651725769, 0.4356192648410797], "r_norm": [0.520584225654602, 0.5245216488838196, 0.5380268096923828], "ldr": {"dL_sdil": -0.012444496154785156, "dL_bp": -0.010713472962379456, "ratio": 1.161574421150286}}, {"step": 600, "epoch": 1, "train_loss": 0.22007516026496887, "cos_r_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_innovation_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_apical_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "cos_Ac_negg": [0.548774242401123, 0.5115599632263184, 0.42728766798973083], "r_norm": [0.5072118639945984, 0.507872462272644, 0.5214201211929321]}, {"step": 700, "epoch": 1, "train_loss": 0.10645534098148346, "cos_r_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_innovation_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_apical_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "cos_Ac_negg": [0.5577744841575623, 0.5250405073165894, 0.4485853910446167], "r_norm": [0.3919033408164978, 0.3957861661911011, 0.4059203267097473]}, {"step": 800, "epoch": 1, "train_loss": 0.099101722240448, "cos_r_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_innovation_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_apical_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "cos_Ac_negg": [0.5633866786956787, 0.5421226024627686, 0.4494962692260742], "r_norm": [0.35605648159980774, 0.3594273328781128, 0.36916738748550415]}, {"step": 900, "epoch": 1, "train_loss": 0.11939629912376404, "cos_r_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_innovation_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_apical_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "cos_Ac_negg": [0.580341100692749, 0.5596993565559387, 0.4562115967273712], "r_norm": [0.35462111234664917, 0.356706440448761, 0.3648630678653717]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9565, "test_loss": 0.1442838306427002}, {"step": 1000, "epoch": 2, "train_loss": 0.10045582056045532, "cos_r_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_innovation_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_apical_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "cos_Ac_negg": [0.5762250423431396, 0.565385103225708, 0.46840357780456543], "r_norm": [0.28615742921829224, 0.288568913936615, 0.29509449005126953], "ldr": {"dL_sdil": -0.0052221789956092834, "dL_bp": -0.004839010536670685, "ratio": 1.0791832247594617}}, {"step": 1100, "epoch": 2, "train_loss": 0.09472673386335373, "cos_r_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_innovation_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_apical_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "cos_Ac_negg": [0.5664094686508179, 0.5700337886810303, 0.48000356554985046], "r_norm": [0.29523903131484985, 0.295265257358551, 0.30285394191741943]}, {"step": 1200, "epoch": 2, "train_loss": 0.02559100277721882, "cos_r_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_innovation_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_apical_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "cos_Ac_negg": [0.5798338651657104, 0.5716899633407593, 0.49267521500587463], "r_norm": [0.30911892652511597, 0.31053924560546875, 0.3160984516143799]}, {"step": 1300, "epoch": 2, "train_loss": 0.15685486793518066, "cos_r_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_innovation_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_apical_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "cos_Ac_negg": [0.5904578566551208, 0.59508216381073, 0.4969713091850281], "r_norm": [0.2874014675617218, 0.290073037147522, 0.2968340814113617]}, {"step": 1400, "epoch": 2, "train_loss": 0.23597419261932373, "cos_r_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_innovation_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_apical_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "cos_Ac_negg": [0.573271632194519, 0.5929781198501587, 0.4888983964920044], "r_norm": [0.3454799950122833, 0.34709277749061584, 0.35570353269577026]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9584, "test_loss": 0.1362682201385498}, {"step": 1500, "epoch": 3, "train_loss": 0.049877192825078964, "cos_r_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_innovation_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_apical_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "cos_Ac_negg": [0.5788212418556213, 0.5889820456504822, 0.49286848306655884], "r_norm": [0.32026493549346924, 0.32111209630966187, 0.32673612236976624], "ldr": {"dL_sdil": -0.008508473634719849, "dL_bp": -0.007917135953903198, "ratio": 1.0746908584442227}}, {"step": 1600, "epoch": 3, "train_loss": 0.04681463539600372, "cos_r_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_innovation_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_apical_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "cos_Ac_negg": [0.5812249779701233, 0.5933512449264526, 0.515969455242157], "r_norm": [0.2679218351840973, 0.2675299048423767, 0.2748056650161743]}, {"step": 1700, "epoch": 3, "train_loss": 0.042607326060533524, "cos_r_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_innovation_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_apical_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "cos_Ac_negg": [0.592969536781311, 0.5956996083259583, 0.5169702768325806], "r_norm": [0.25324034690856934, 0.25399982929229736, 0.25864163041114807]}, {"step": 1800, "epoch": 3, "train_loss": 0.05712303891777992, "cos_r_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_innovation_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_apical_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "cos_Ac_negg": [0.5795919895172119, 0.6103173494338989, 0.5235826969146729], "r_norm": [0.26621386408805847, 0.2666720151901245, 0.27278897166252136]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9686, "test_loss": 0.10741588859558106}, {"step": 1900, "epoch": 4, "train_loss": 0.0610680989921093, "cos_r_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_innovation_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_apical_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "cos_Ac_negg": [0.5814831256866455, 0.5985654592514038, 0.5398411154747009], "r_norm": [0.255119651556015, 0.25568583607673645, 0.26110944151878357]}, {"step": 2000, "epoch": 4, "train_loss": 0.059706609696149826, "cos_r_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_innovation_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_apical_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "cos_Ac_negg": [0.5836533308029175, 0.6085301637649536, 0.5367716550827026], "r_norm": [0.21822944283485413, 0.2185012400150299, 0.22326743602752686], "ldr": {"dL_sdil": -0.004507958889007568, "dL_bp": -0.004968710243701935, "ratio": 0.9072694256465469}}, {"step": 2100, "epoch": 4, "train_loss": 0.032644227147102356, "cos_r_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_innovation_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_apical_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "cos_Ac_negg": [0.5811567306518555, 0.6085543632507324, 0.5547049641609192], "r_norm": [0.24712875485420227, 0.24843883514404297, 0.2534225881099701]}, {"step": 2200, "epoch": 4, "train_loss": 0.028950463980436325, "cos_r_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_innovation_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_apical_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "cos_Ac_negg": [0.5847131013870239, 0.6218141913414001, 0.5643103122711182], "r_norm": [0.24966076016426086, 0.2504446804523468, 0.25478261709213257]}, {"step": 2300, "epoch": 4, "train_loss": 0.0650198757648468, "cos_r_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_innovation_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_apical_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "cos_Ac_negg": [0.5730236768722534, 0.6205640435218811, 0.5527874231338501], "r_norm": [0.20470218360424042, 0.2031494677066803, 0.20760329067707062]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9693, "test_loss": 0.10958373928070068}, {"step": 2400, "epoch": 5, "train_loss": 0.03290189430117607, "cos_r_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_innovation_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_apical_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "cos_Ac_negg": [0.5914133787155151, 0.6294198632240295, 0.5489491820335388], "r_norm": [0.21331188082695007, 0.21278373897075653, 0.2184755653142929]}, {"step": 2500, "epoch": 5, "train_loss": 0.03874827176332474, "cos_r_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_innovation_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_apical_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "cos_Ac_negg": [0.5949760675430298, 0.6306216716766357, 0.5710424780845642], "r_norm": [0.22789107263088226, 0.2283928245306015, 0.23281589150428772], "ldr": {"dL_sdil": -0.0070598796010017395, "dL_bp": -0.008158765733242035, "ratio": 0.8653122092030416}}, {"step": 2600, "epoch": 5, "train_loss": 0.021128688007593155, "cos_r_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_innovation_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_apical_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "cos_Ac_negg": [0.5858960151672363, 0.6278800368309021, 0.5700942873954773], "r_norm": [0.2212328165769577, 0.2204180359840393, 0.2251305729150772]}, {"step": 2700, "epoch": 5, "train_loss": 0.07236946374177933, "cos_r_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_innovation_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_apical_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "cos_Ac_negg": [0.6011340618133545, 0.6260433197021484, 0.570478081703186], "r_norm": [0.2149030566215515, 0.2140052616596222, 0.2162187099456787]}, {"step": 2800, "epoch": 5, "train_loss": 0.008522935211658478, "cos_r_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_innovation_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_apical_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "cos_Ac_negg": [0.5996780395507812, 0.627189576625824, 0.5757463574409485], "r_norm": [0.205214723944664, 0.2052396833896637, 0.2100837081670761]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9724, "test_loss": 0.10797543697357177}, {"step": 2900, "epoch": 6, "train_loss": 0.018756665289402008, "cos_r_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_innovation_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_apical_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "cos_Ac_negg": [0.6026749610900879, 0.6295627355575562, 0.5728473663330078], "r_norm": [0.22938458621501923, 0.22977539896965027, 0.23387743532657623]}, {"step": 3000, "epoch": 6, "train_loss": 0.005393678322434425, "cos_r_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_innovation_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_apical_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "cos_Ac_negg": [0.5884539484977722, 0.6252824664115906, 0.5843188166618347], "r_norm": [0.19885726273059845, 0.1973133385181427, 0.20168818533420563], "ldr": {"dL_sdil": -0.005929701030254364, "dL_bp": -0.008110091090202332, "ratio": 0.7311509777514015}}, {"step": 3100, "epoch": 6, "train_loss": 0.03258326277136803, "cos_r_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_innovation_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_apical_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "cos_Ac_negg": [0.5935311317443848, 0.6407343745231628, 0.5825847387313843], "r_norm": [0.22770541906356812, 0.22627578675746918, 0.22949440777301788]}, {"step": 3200, "epoch": 6, "train_loss": 0.05724141746759415, "cos_r_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_innovation_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_apical_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "cos_Ac_negg": [0.5926040410995483, 0.6424926519393921, 0.5951640605926514], "r_norm": [0.23419582843780518, 0.23333817720413208, 0.23697063326835632]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9735, "test_loss": 0.1085232892036438}, {"step": 3300, "epoch": 7, "train_loss": 0.044839728623628616, "cos_r_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_innovation_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_apical_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "cos_Ac_negg": [0.5871893167495728, 0.6351736783981323, 0.5917670726776123], "r_norm": [0.18712347745895386, 0.185409277677536, 0.1905364990234375]}, {"step": 3400, "epoch": 7, "train_loss": 0.008097128942608833, "cos_r_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_innovation_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_apical_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "cos_Ac_negg": [0.598461389541626, 0.6425559520721436, 0.5985062122344971], "r_norm": [0.19379451870918274, 0.19184225797653198, 0.19568520784378052]}, {"step": 3500, "epoch": 7, "train_loss": 0.010430501773953438, "cos_r_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_innovation_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_apical_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "cos_Ac_negg": [0.5999592542648315, 0.6464356780052185, 0.6043606996536255], "r_norm": [0.1839253008365631, 0.18301129341125488, 0.18623384833335876], "ldr": {"dL_sdil": -0.006262347102165222, "dL_bp": -0.009936518967151642, "ratio": 0.6302355103298675}}, {"step": 3600, "epoch": 7, "train_loss": 0.0012780515244230628, "cos_r_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_innovation_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_apical_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "cos_Ac_negg": [0.5982638597488403, 0.6448532342910767, 0.6086806654930115], "r_norm": [0.1790696531534195, 0.1764722466468811, 0.17974436283111572]}, {"step": 3700, "epoch": 7, "train_loss": 0.018456075340509415, "cos_r_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_innovation_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_apical_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "cos_Ac_negg": [0.596823513507843, 0.6471102237701416, 0.6086265444755554], "r_norm": [0.18262530863285065, 0.18070702254772186, 0.1839236468076706]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.973, "test_loss": 0.11429949645996093}, {"step": 3800, "epoch": 8, "train_loss": 0.019817866384983063, "cos_r_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_innovation_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_apical_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "cos_Ac_negg": [0.5917383432388306, 0.6484235525131226, 0.6023948192596436], "r_norm": [0.1908964067697525, 0.19014066457748413, 0.19408388435840607]}, {"step": 3900, "epoch": 8, "train_loss": 0.019774047657847404, "cos_r_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_innovation_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_apical_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "cos_Ac_negg": [0.6020888090133667, 0.6518036723136902, 0.6111307740211487], "r_norm": [0.19253888726234436, 0.19145795702934265, 0.19445383548736572]}, {"step": 4000, "epoch": 8, "train_loss": 0.0010947450064122677, "cos_r_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_innovation_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_apical_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "cos_Ac_negg": [0.5878117680549622, 0.6392156481742859, 0.6161020398139954], "r_norm": [0.18950730562210083, 0.18815365433692932, 0.19119828939437866], "ldr": {"dL_sdil": -0.007813781499862671, "dL_bp": -0.014065638184547424, "ratio": 0.5555227141024378}}, {"step": 4100, "epoch": 8, "train_loss": 0.09640847891569138, "cos_r_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_innovation_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_apical_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "cos_Ac_negg": [0.5970286130905151, 0.6522461175918579, 0.6217045783996582], "r_norm": [0.1922830492258072, 0.19028913974761963, 0.19349952042102814]}, {"step": 4200, "epoch": 8, "train_loss": 0.014551833271980286, "cos_r_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_innovation_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_apical_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "cos_Ac_negg": [0.5896190404891968, 0.6460390090942383, 0.6227425336837769], "r_norm": [0.19427847862243652, 0.19271467626094818, 0.19574645161628723]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9729, "test_loss": 0.12127476367950439}, {"step": 4300, "epoch": 9, "train_loss": 0.0006672337185591459, "cos_r_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_innovation_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_apical_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "cos_Ac_negg": [0.5907028913497925, 0.6471917629241943, 0.6189355850219727], "r_norm": [0.19403856992721558, 0.19274917244911194, 0.1965358704328537]}, {"step": 4400, "epoch": 9, "train_loss": 0.004814686719328165, "cos_r_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_innovation_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_apical_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "cos_Ac_negg": [0.5863927006721497, 0.6445280909538269, 0.6254445314407349], "r_norm": [0.1811094731092453, 0.1802864819765091, 0.18380987644195557]}, {"step": 4500, "epoch": 9, "train_loss": 0.007938915863633156, "cos_r_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_innovation_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_apical_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "cos_Ac_negg": [0.5819159746170044, 0.6380576491355896, 0.6219855546951294], "r_norm": [0.18736639618873596, 0.185921311378479, 0.1901116669178009], "ldr": {"dL_sdil": -0.008901968598365784, "dL_bp": -0.017541170120239258, "ratio": 0.5074900099221181}}, {"step": 4600, "epoch": 9, "train_loss": 0.0795854851603508, "cos_r_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_innovation_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_apical_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "cos_Ac_negg": [0.5859084725379944, 0.6494425535202026, 0.6282416582107544], "r_norm": [0.184382826089859, 0.1827830672264099, 0.18596649169921875]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9742, "test_loss": 0.12309427490234374}, {"step": 4700, "epoch": 10, "train_loss": 0.0007766007911413908, "cos_r_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_innovation_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_apical_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "cos_Ac_negg": [0.5853949785232544, 0.6475309133529663, 0.6286728382110596], "r_norm": [0.19498443603515625, 0.19344991445541382, 0.19718916714191437]}, {"step": 4800, "epoch": 10, "train_loss": 0.002889199648052454, "cos_r_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_innovation_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_apical_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "cos_Ac_negg": [0.5899951457977295, 0.6545200347900391, 0.6340373754501343], "r_norm": [0.19870562851428986, 0.1971784234046936, 0.20026731491088867]}, {"step": 4900, "epoch": 10, "train_loss": 0.0008581852307543159, "cos_r_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_innovation_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_apical_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "cos_Ac_negg": [0.5852946043014526, 0.6489967107772827, 0.6322373747825623], "r_norm": [0.19465115666389465, 0.19327929615974426, 0.19643467664718628]}, {"step": 5000, "epoch": 10, "train_loss": 0.0007646966842003167, "cos_r_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_innovation_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_apical_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "cos_Ac_negg": [0.5893222093582153, 0.6465706825256348, 0.6309077739715576], "r_norm": [0.19242608547210693, 0.19134974479675293, 0.19527661800384521], "ldr": {"dL_sdil": -0.009105280041694641, "dL_bp": -0.018391385674476624, "ratio": 0.4950839595697705}}, {"step": 5100, "epoch": 10, "train_loss": 0.0015073801623657346, "cos_r_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_innovation_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_apical_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "cos_Ac_negg": [0.5857441425323486, 0.6448982954025269, 0.6283546090126038], "r_norm": [0.17740818858146667, 0.17602333426475525, 0.17976273596286774]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9745, "test_loss": 0.12885003728866576}, {"step": 5200, "epoch": 11, "train_loss": 0.0014642335008829832, "cos_r_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_innovation_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_apical_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "cos_Ac_negg": [0.5893962383270264, 0.6453257203102112, 0.6312963962554932], "r_norm": [0.17490693926811218, 0.17364893853664398, 0.17717021703720093]}, {"step": 5300, "epoch": 11, "train_loss": 0.001281333388760686, "cos_r_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_innovation_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_apical_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "cos_Ac_negg": [0.5883110761642456, 0.6459683775901794, 0.6333180069923401], "r_norm": [0.1789165437221527, 0.1773441880941391, 0.1806153953075409]}, {"step": 5400, "epoch": 11, "train_loss": 0.0045455265790224075, "cos_r_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_innovation_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_apical_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "cos_Ac_negg": [0.5888869762420654, 0.6436636447906494, 0.6318002939224243], "r_norm": [0.18679404258728027, 0.18510667979717255, 0.18804368376731873]}, {"step": 5500, "epoch": 11, "train_loss": 0.002247859025374055, "cos_r_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_innovation_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_apical_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "cos_Ac_negg": [0.5850186347961426, 0.641714870929718, 0.6345680952072144], "r_norm": [0.1849762499332428, 0.18319714069366455, 0.18662986159324646], "ldr": {"dL_sdil": -0.00928555428981781, "dL_bp": -0.02012455463409424, "ratio": 0.4614042128458627}}, {"step": 5600, "epoch": 11, "train_loss": 0.0012828883482143283, "cos_r_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_innovation_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_apical_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "cos_Ac_negg": [0.5890182256698608, 0.6434066295623779, 0.6389946341514587], "r_norm": [0.1720443069934845, 0.1705038845539093, 0.17383548617362976]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9739, "test_loss": 0.1312964542388916}, {"step": 5700, "epoch": 12, "train_loss": 0.0005914976936765015, "cos_r_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_innovation_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_apical_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "cos_Ac_negg": [0.5882803201675415, 0.6447182297706604, 0.6371629238128662], "r_norm": [0.18014508485794067, 0.1789059042930603, 0.18226537108421326]}, {"step": 5800, "epoch": 12, "train_loss": 0.0010177994845435023, "cos_r_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_innovation_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_apical_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "cos_Ac_negg": [0.5830683708190918, 0.6426451206207275, 0.636396586894989], "r_norm": [0.19342085719108582, 0.19208499789237976, 0.19565963745117188]}, {"step": 5900, "epoch": 12, "train_loss": 0.0026742268819361925, "cos_r_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_innovation_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_apical_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "cos_Ac_negg": [0.5833733081817627, 0.6423500180244446, 0.6389040946960449], "r_norm": [0.18534237146377563, 0.18388569355010986, 0.18735302984714508]}, {"step": 6000, "epoch": 12, "train_loss": 0.0006315991631709039, "cos_r_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_innovation_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_apical_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "cos_Ac_negg": [0.5834283232688904, 0.6447769999504089, 0.6399239301681519], "r_norm": [0.18172697722911835, 0.1802336871623993, 0.18384242057800293], "ldr": {"dL_sdil": -0.009645640850067139, "dL_bp": -0.021210603415966034, "ratio": 0.4547556078865015}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9741, "test_loss": 0.13412653560638427}, {"step": 6100, "epoch": 13, "train_loss": 0.0009790371404960752, "cos_r_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_innovation_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_apical_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "cos_Ac_negg": [0.5813207030296326, 0.642903745174408, 0.6394165754318237], "r_norm": [0.18465885519981384, 0.18316440284252167, 0.18671941757202148]}, {"step": 6200, "epoch": 13, "train_loss": 0.0008633113466203213, "cos_r_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_innovation_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_apical_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "cos_Ac_negg": [0.5780510306358337, 0.6401042938232422, 0.6404061317443848], "r_norm": [0.18658879399299622, 0.18528807163238525, 0.18887931108474731]}, {"step": 6300, "epoch": 13, "train_loss": 0.0002778768539428711, "cos_r_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_innovation_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_apical_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "cos_Ac_negg": [0.5788507461547852, 0.6427652835845947, 0.6413661241531372], "r_norm": [0.18958455324172974, 0.188012033700943, 0.19160720705986023]}, {"step": 6400, "epoch": 13, "train_loss": 0.00041367538506165147, "cos_r_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_innovation_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_apical_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "cos_Ac_negg": [0.5766105055809021, 0.6364271640777588, 0.6403087377548218], "r_norm": [0.19554319977760315, 0.1940520703792572, 0.19749662280082703]}, {"step": 6500, "epoch": 13, "train_loss": 0.00021037989063188434, "cos_r_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_innovation_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_apical_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "cos_Ac_negg": [0.5767596364021301, 0.6377795338630676, 0.640342652797699], "r_norm": [0.18189536035060883, 0.18059924244880676, 0.18395429849624634], "ldr": {"dL_sdil": -0.009790733456611633, "dL_bp": -0.022421933710575104, "ratio": 0.43665874598469273}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9738, "test_loss": 0.1366198040008545}, {"step": 6600, "epoch": 14, "train_loss": 0.0006715196650475264, "cos_r_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_innovation_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_apical_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "cos_Ac_negg": [0.5768696069717407, 0.6378884315490723, 0.6428689956665039], "r_norm": [0.17912247776985168, 0.17778383195400238, 0.18115678429603577]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008008020813576877, "cos_r_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_innovation_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_apical_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "cos_Ac_negg": [0.5771304368972778, 0.6388176679611206, 0.6439354419708252], "r_norm": [0.17753815650939941, 0.17617246508598328, 0.1796029806137085]}, {"step": 6800, "epoch": 14, "train_loss": 0.0007890297565609217, "cos_r_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_innovation_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_apical_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "cos_Ac_negg": [0.5742036700248718, 0.6357213258743286, 0.6409169435501099], "r_norm": [0.18021561205387115, 0.1791214942932129, 0.18264533579349518]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005114716477692127, "cos_r_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_innovation_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_apical_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "cos_Ac_negg": [0.5761258602142334, 0.6392124891281128, 0.6455451250076294], "r_norm": [0.17889344692230225, 0.17748130857944489, 0.18085458874702454]}, {"step": 7000, "epoch": 14, "train_loss": 0.0008774346788413823, "cos_r_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_innovation_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_apical_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "cos_Ac_negg": [0.5758652687072754, 0.638709545135498, 0.6432210206985474], "r_norm": [0.18395139276981354, 0.18258227407932281, 0.18588440120220184], "ldr": {"dL_sdil": -0.010152354836463928, "dL_bp": -0.023831740021705627, "ratio": 0.4260014093480921}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9743, "test_loss": 0.13829146881103516}], "final": {"test_acc": 0.9743, "test_loss": 0.13829146881103516, "wall_s": 20.234329223632812, "cos_r_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_innovation_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_apical_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716], "cos_Ac_negg": [0.5751822590827942, 0.6385658979415894, 0.6445321440696716]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_matched_s0.json b/results/nuis_ctrl_v1_rho0p05_matched_s0.json new file mode 100644 index 0000000..84f50f6 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_matched_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_matched_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.018888577818870544, -0.025971781462430954, 0.032935731112957], "cos_innovation_negg": [-0.01886148937046528, -0.025820758193731308, 0.03358188271522522], "cos_apical_negg": [-0.018888579681515694, -0.025971779599785805, 0.0329357348382473], "cos_Ac_negg": [-0.01886145956814289, -0.025820761919021606, 0.033581867814064026], "r_norm": [4.897843360900879, 4.826535701751709, 4.823820114135742], "ldr": {"dL_sdil": -0.0094451904296875, "dL_bp": -0.23505353927612305, "ratio": 0.04018314490722051}}, {"step": 100, "epoch": 0, "train_loss": 0.2925134003162384, "cos_r_negg": [0.2600802779197693, 0.26295357942581177, 0.26131099462509155], "cos_innovation_negg": [0.5080794095993042, 0.47609972953796387, 0.4329450726509094], "cos_apical_negg": [0.2600802779197693, 0.26295357942581177, 0.26131099462509155], "cos_Ac_negg": [0.5081974267959595, 0.47605615854263306, 0.4330165386199951], "r_norm": [1.0393927097320557, 1.017633080482483, 1.0119723081588745]}, {"step": 200, "epoch": 0, "train_loss": 0.35974711179733276, "cos_r_negg": [0.21381282806396484, 0.21494406461715698, 0.2353992760181427], "cos_innovation_negg": [0.5576655864715576, 0.5311211347579956, 0.4876919090747833], "cos_apical_negg": [0.21381282806396484, 0.21494406461715698, 0.2353992760181427], "cos_Ac_negg": [0.5579656362533569, 0.5309754014015198, 0.4882020056247711], "r_norm": [0.7699924111366272, 0.7572832107543945, 0.7494314312934875]}, {"step": 300, "epoch": 0, "train_loss": 0.2397359162569046, "cos_r_negg": [0.20823854207992554, 0.21287313103675842, 0.21960140764713287], "cos_innovation_negg": [0.5615941286087036, 0.5571704506874084, 0.51097571849823], "cos_apical_negg": [0.20823854207992554, 0.21287314593791962, 0.21960140764713287], "cos_Ac_negg": [0.5611709356307983, 0.5571225881576538, 0.5108929872512817], "r_norm": [0.636157751083374, 0.622569739818573, 0.6193011999130249]}, {"step": 400, "epoch": 0, "train_loss": 0.16822239756584167, "cos_r_negg": [0.24075540900230408, 0.2516685128211975, 0.23215335607528687], "cos_innovation_negg": [0.5859761238098145, 0.5680060386657715, 0.573615550994873], "cos_apical_negg": [0.24075540900230408, 0.2516685128211975, 0.23215335607528687], "cos_Ac_negg": [0.585658848285675, 0.5683999061584473, 0.5734416246414185], "r_norm": [0.7184232473373413, 0.7055515050888062, 0.7036710977554321]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9367, "test_loss": 0.20731257934570313}, {"step": 500, "epoch": 1, "train_loss": 0.2124519646167755, "cos_r_negg": [0.19893747568130493, 0.21118637919425964, 0.21870911121368408], "cos_innovation_negg": [0.5929887294769287, 0.5930097103118896, 0.5744439363479614], "cos_apical_negg": [0.19893747568130493, 0.21118637919425964, 0.21870911121368408], "cos_Ac_negg": [0.5926508903503418, 0.5930609703063965, 0.5743594169616699], "r_norm": [0.5818136930465698, 0.5715469121932983, 0.569089412689209], "ldr": {"dL_sdil": -0.01115541160106659, "dL_bp": -0.010585024952888489, "ratio": 1.0538861883384083}}, {"step": 600, "epoch": 1, "train_loss": 0.09760530292987823, "cos_r_negg": [0.1724637746810913, 0.1980946958065033, 0.21229907870292664], "cos_innovation_negg": [0.5967092514038086, 0.5914995670318604, 0.6301769018173218], "cos_apical_negg": [0.17246375977993011, 0.1980946958065033, 0.21229907870292664], "cos_Ac_negg": [0.5966199636459351, 0.5912399291992188, 0.6300961971282959], "r_norm": [0.5214858651161194, 0.5144153237342834, 0.5116932392120361]}, {"step": 700, "epoch": 1, "train_loss": 0.2011653482913971, "cos_r_negg": [0.18453750014305115, 0.2045569121837616, 0.21993455290794373], "cos_innovation_negg": [0.5839407444000244, 0.5973491668701172, 0.6437468528747559], "cos_apical_negg": [0.18453750014305115, 0.2045568972826004, 0.21993455290794373], "cos_Ac_negg": [0.5839197039604187, 0.5971819162368774, 0.643746018409729], "r_norm": [0.47424277663230896, 0.46459078788757324, 0.4637201726436615]}, {"step": 800, "epoch": 1, "train_loss": 0.17559540271759033, "cos_r_negg": [0.1664370447397232, 0.1795715093612671, 0.18634530901908875], "cos_innovation_negg": [0.5826842188835144, 0.6021206378936768, 0.6356761455535889], "cos_apical_negg": [0.1664370447397232, 0.17957152426242828, 0.18634530901908875], "cos_Ac_negg": [0.5825594067573547, 0.6016643643379211, 0.6353830099105835], "r_norm": [0.49542903900146484, 0.4850253462791443, 0.48303377628326416]}, {"step": 900, "epoch": 1, "train_loss": 0.16637590527534485, "cos_r_negg": [0.16264620423316956, 0.1967795044183731, 0.2149035483598709], "cos_innovation_negg": [0.5857820510864258, 0.5955974459648132, 0.6671602725982666], "cos_apical_negg": [0.16264620423316956, 0.1967795193195343, 0.2149035632610321], "cos_Ac_negg": [0.5857768654823303, 0.5954445600509644, 0.6669906377792358], "r_norm": [0.4447886347770691, 0.43750229477882385, 0.4374445974826813]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9507, "test_loss": 0.18049224014282225}, {"step": 1000, "epoch": 2, "train_loss": 0.07176452875137329, "cos_r_negg": [0.19881696999073029, 0.23217740654945374, 0.23776540160179138], "cos_innovation_negg": [0.5895395278930664, 0.6104155778884888, 0.6908552646636963], "cos_apical_negg": [0.19881696999073029, 0.23217740654945374, 0.23776540160179138], "cos_Ac_negg": [0.5894551277160645, 0.6103118062019348, 0.690676212310791], "r_norm": [0.5243955850601196, 0.518196702003479, 0.5109337568283081], "ldr": {"dL_sdil": -0.024999797344207764, "dL_bp": -0.024990737438201904, "ratio": 1.0003625305587025}}, {"step": 1100, "epoch": 2, "train_loss": 0.21467101573944092, "cos_r_negg": [0.14204896986484528, 0.175796777009964, 0.20578661561012268], "cos_innovation_negg": [0.5654441714286804, 0.5900775194168091, 0.668527364730835], "cos_apical_negg": [0.14204896986484528, 0.175796777009964, 0.20578661561012268], "cos_Ac_negg": [0.5649679899215698, 0.5897566080093384, 0.6681588292121887], "r_norm": [0.40868109464645386, 0.40163397789001465, 0.40070417523384094]}, {"step": 1200, "epoch": 2, "train_loss": 0.23202364146709442, "cos_r_negg": [0.14741116762161255, 0.1883571445941925, 0.21747739613056183], "cos_innovation_negg": [0.573001503944397, 0.6010042428970337, 0.6759384870529175], "cos_apical_negg": [0.14741116762161255, 0.1883571445941925, 0.21747739613056183], "cos_Ac_negg": [0.5726856589317322, 0.6007051467895508, 0.675615668296814], "r_norm": [0.40492138266563416, 0.3971380889415741, 0.3964496850967407]}, {"step": 1300, "epoch": 2, "train_loss": 0.14806914329528809, "cos_r_negg": [0.12889954447746277, 0.1617678552865982, 0.2525172233581543], "cos_innovation_negg": [0.5734646320343018, 0.5951402187347412, 0.6983329653739929], "cos_apical_negg": [0.12889954447746277, 0.1617678552865982, 0.2525172233581543], "cos_Ac_negg": [0.573162317276001, 0.5947151780128479, 0.697583794593811], "r_norm": [0.4237639307975769, 0.41418731212615967, 0.4155843257904053]}, {"step": 1400, "epoch": 2, "train_loss": 0.17553207278251648, "cos_r_negg": [0.13460661470890045, 0.15178261697292328, 0.2361353039741516], "cos_innovation_negg": [0.5718982815742493, 0.6067863702774048, 0.7094591856002808], "cos_apical_negg": [0.13460661470890045, 0.15178261697292328, 0.23613528907299042], "cos_Ac_negg": [0.5713673830032349, 0.6060904860496521, 0.708818256855011], "r_norm": [0.36141514778137207, 0.3557671308517456, 0.3550999164581299]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9538, "test_loss": 0.1557269275665283}, {"step": 1500, "epoch": 3, "train_loss": 0.13190995156764984, "cos_r_negg": [0.13948999345302582, 0.15122002363204956, 0.17426201701164246], "cos_innovation_negg": [0.5648857355117798, 0.6017195582389832, 0.7207717895507812], "cos_apical_negg": [0.13948999345302582, 0.15122002363204956, 0.17426201701164246], "cos_Ac_negg": [0.5642306804656982, 0.6009235382080078, 0.7202455401420593], "r_norm": [0.3556489050388336, 0.3492661416530609, 0.34925156831741333], "ldr": {"dL_sdil": -0.006761610507965088, "dL_bp": -0.007895633578300476, "ratio": 0.8563733918134173}}, {"step": 1600, "epoch": 3, "train_loss": 0.12020806968212128, "cos_r_negg": [0.1394944041967392, 0.1747950315475464, 0.22166553139686584], "cos_innovation_negg": [0.5503301024436951, 0.6051430106163025, 0.7060390114784241], "cos_apical_negg": [0.1394944190979004, 0.1747950315475464, 0.22166553139686584], "cos_Ac_negg": [0.5502285361289978, 0.6051437854766846, 0.7059417963027954], "r_norm": [0.3799726665019989, 0.37381744384765625, 0.37025994062423706]}, {"step": 1700, "epoch": 3, "train_loss": 0.12640194594860077, "cos_r_negg": [0.13853128254413605, 0.17026695609092712, 0.192499577999115], "cos_innovation_negg": [0.5643026828765869, 0.614161491394043, 0.7227206230163574], "cos_apical_negg": [0.13853129744529724, 0.17026695609092712, 0.192499577999115], "cos_Ac_negg": [0.564063549041748, 0.6139358282089233, 0.722440242767334], "r_norm": [0.41324523091316223, 0.40703123807907104, 0.4055345356464386]}, {"step": 1800, "epoch": 3, "train_loss": 0.021324677392840385, "cos_r_negg": [0.11448002606630325, 0.1622004359960556, 0.15379531681537628], "cos_innovation_negg": [0.5607688426971436, 0.5947904586791992, 0.7067182064056396], "cos_apical_negg": [0.11448002606630325, 0.1622004210948944, 0.15379533171653748], "cos_Ac_negg": [0.560477614402771, 0.594211220741272, 0.7057375907897949], "r_norm": [0.34976544976234436, 0.34635043144226074, 0.3432033061981201]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9589, "test_loss": 0.13953808059692382}, {"step": 1900, "epoch": 4, "train_loss": 0.08204656839370728, "cos_r_negg": [0.11766695976257324, 0.15146785974502563, 0.16843679547309875], "cos_innovation_negg": [0.5684402585029602, 0.5953737497329712, 0.7274236679077148], "cos_apical_negg": [0.11766695976257324, 0.15146785974502563, 0.16843679547309875], "cos_Ac_negg": [0.5680156350135803, 0.5950276851654053, 0.7267047166824341], "r_norm": [0.3084789514541626, 0.30436310172080994, 0.3013650178909302]}, {"step": 2000, "epoch": 4, "train_loss": 0.16747495532035828, "cos_r_negg": [0.09503549337387085, 0.13075682520866394, 0.19327878952026367], "cos_innovation_negg": [0.5502690076828003, 0.5977407693862915, 0.7345064878463745], "cos_apical_negg": [0.09503549337387085, 0.13075682520866394, 0.19327880442142487], "cos_Ac_negg": [0.5494052171707153, 0.597416877746582, 0.7338563799858093], "r_norm": [0.33181628584861755, 0.3239547610282898, 0.32429227232933044], "ldr": {"dL_sdil": -0.011301577091217041, "dL_bp": -0.014433056116104126, "ratio": 0.7830342375380194}}, {"step": 2100, "epoch": 4, "train_loss": 0.0479811355471611, "cos_r_negg": [0.1210709661245346, 0.16421130299568176, 0.21043890714645386], "cos_innovation_negg": [0.5522477626800537, 0.5935909748077393, 0.7170997262001038], "cos_apical_negg": [0.1210709661245346, 0.16421130299568176, 0.21043890714645386], "cos_Ac_negg": [0.5515713691711426, 0.5923702716827393, 0.7165814638137817], "r_norm": [0.4709198772907257, 0.46099433302879333, 0.4572564959526062]}, {"step": 2200, "epoch": 4, "train_loss": 0.030225181952118874, "cos_r_negg": [0.10094594210386276, 0.15073946118354797, 0.1727372705936432], "cos_innovation_negg": [0.5547534227371216, 0.6040734648704529, 0.7391708493232727], "cos_apical_negg": [0.10094594210386276, 0.15073946118354797, 0.172737255692482], "cos_Ac_negg": [0.5542435050010681, 0.6037734746932983, 0.7387421131134033], "r_norm": [0.30935361981391907, 0.3030080497264862, 0.3018103241920471]}, {"step": 2300, "epoch": 4, "train_loss": 0.22975580394268036, "cos_r_negg": [0.11159854382276535, 0.14088667929172516, 0.16555047035217285], "cos_innovation_negg": [0.5509880185127258, 0.5912683606147766, 0.7330929040908813], "cos_apical_negg": [0.11159853637218475, 0.14088669419288635, 0.16555047035217285], "cos_Ac_negg": [0.550560474395752, 0.5900709629058838, 0.7318021059036255], "r_norm": [0.2574431002140045, 0.25378215312957764, 0.2512046694755554]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9608, "test_loss": 0.13752566566467286}, {"step": 2400, "epoch": 5, "train_loss": 0.13832847774028778, "cos_r_negg": [0.09531106054782867, 0.11300396919250488, 0.17430379986763], "cos_innovation_negg": [0.5580445528030396, 0.6096996665000916, 0.732465386390686], "cos_apical_negg": [0.09531106799840927, 0.11300396919250488, 0.17430379986763], "cos_Ac_negg": [0.5576131343841553, 0.6092826128005981, 0.731804370880127], "r_norm": [0.29090115427970886, 0.286607027053833, 0.284349262714386]}, {"step": 2500, "epoch": 5, "train_loss": 0.03487436845898628, "cos_r_negg": [0.08470235764980316, 0.09950131922960281, 0.13054385781288147], "cos_innovation_negg": [0.5414856672286987, 0.5867946147918701, 0.7511945962905884], "cos_apical_negg": [0.08470235764980316, 0.09950131922960281, 0.13054384291172028], "cos_Ac_negg": [0.5413762927055359, 0.5864846706390381, 0.7509655952453613], "r_norm": [0.27695605158805847, 0.27181476354599, 0.27067533135414124], "ldr": {"dL_sdil": -0.00807391107082367, "dL_bp": -0.010298319160938263, "ratio": 0.7840028012967573}}, {"step": 2600, "epoch": 5, "train_loss": 0.11033789813518524, "cos_r_negg": [0.0956190824508667, 0.12613743543624878, 0.135524183511734], "cos_innovation_negg": [0.5575433969497681, 0.587949275970459, 0.7775989770889282], "cos_apical_negg": [0.0956190824508667, 0.12613743543624878, 0.135524183511734], "cos_Ac_negg": [0.5576225519180298, 0.5881427526473999, 0.7769110798835754], "r_norm": [0.34104347229003906, 0.3375483453273773, 0.331754595041275]}, {"step": 2700, "epoch": 5, "train_loss": 0.04781732335686684, "cos_r_negg": [0.08584916591644287, 0.11772655695676804, 0.16713127493858337], "cos_innovation_negg": [0.5652093291282654, 0.5992367267608643, 0.7422999739646912], "cos_apical_negg": [0.08584915846586227, 0.11772656440734863, 0.16713127493858337], "cos_Ac_negg": [0.5655986070632935, 0.5995399951934814, 0.741814136505127], "r_norm": [0.28275835514068604, 0.2767255902290344, 0.2750614881515503]}, {"step": 2800, "epoch": 5, "train_loss": 0.1706671565771103, "cos_r_negg": [0.09122958779335022, 0.11384502053260803, 0.17169907689094543], "cos_innovation_negg": [0.5433820486068726, 0.5899790525436401, 0.761295735836029], "cos_apical_negg": [0.09122960269451141, 0.11384502798318863, 0.17169909179210663], "cos_Ac_negg": [0.5424270629882812, 0.58916836977005, 0.7602348327636719], "r_norm": [0.30017024278640747, 0.2951509356498718, 0.29237961769104004]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9637, "test_loss": 0.13457500228881836}, {"step": 2900, "epoch": 6, "train_loss": 0.019902069121599197, "cos_r_negg": [0.08643680810928345, 0.10017009824514389, 0.16301289200782776], "cos_innovation_negg": [0.5555968880653381, 0.6130666732788086, 0.743310809135437], "cos_apical_negg": [0.08643680065870285, 0.10017010569572449, 0.16301289200782776], "cos_Ac_negg": [0.5552680492401123, 0.6136595010757446, 0.744665265083313], "r_norm": [0.29115843772888184, 0.285186231136322, 0.2850620150566101]}, {"step": 3000, "epoch": 6, "train_loss": 0.1896212100982666, "cos_r_negg": [0.05150953307747841, 0.07955276966094971, 0.12302252650260925], "cos_innovation_negg": [0.556449294090271, 0.6050234436988831, 0.7544301748275757], "cos_apical_negg": [0.05150953680276871, 0.07955276966094971, 0.12302251905202866], "cos_Ac_negg": [0.5569143891334534, 0.606092095375061, 0.7551864385604858], "r_norm": [0.25071385502815247, 0.2459258735179901, 0.2457805871963501], "ldr": {"dL_sdil": -0.0064574480056762695, "dL_bp": -0.00949336588382721, "ratio": 0.6802063761891981}}, {"step": 3100, "epoch": 6, "train_loss": 0.07059909403324127, "cos_r_negg": [0.07147012650966644, 0.10582158714532852, 0.13621792197227478], "cos_innovation_negg": [0.5145161151885986, 0.5923010110855103, 0.7556737661361694], "cos_apical_negg": [0.07147012650966644, 0.10582158714532852, 0.1362179070711136], "cos_Ac_negg": [0.5149269104003906, 0.5934772491455078, 0.7549319267272949], "r_norm": [0.26133793592453003, 0.2572306990623474, 0.2575467824935913]}, {"step": 3200, "epoch": 6, "train_loss": 0.07592140883207321, "cos_r_negg": [0.07781057804822922, 0.09806972742080688, 0.16834115982055664], "cos_innovation_negg": [0.5632877349853516, 0.6036882400512695, 0.7605648040771484], "cos_apical_negg": [0.07781057804822922, 0.09806972742080688, 0.16834115982055664], "cos_Ac_negg": [0.5641144514083862, 0.6079229116439819, 0.7628395557403564], "r_norm": [0.24588164687156677, 0.24154649674892426, 0.23966088891029358]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9658, "test_loss": 0.12357935066223144}, {"step": 3300, "epoch": 7, "train_loss": 0.028484942391514778, "cos_r_negg": [0.06780457496643066, 0.10129158943891525, 0.13399964570999146], "cos_innovation_negg": [0.5625133514404297, 0.5992419719696045, 0.7527547478675842], "cos_apical_negg": [0.06780457496643066, 0.10129158943891525, 0.13399964570999146], "cos_Ac_negg": [0.56358802318573, 0.6007165908813477, 0.753697395324707], "r_norm": [0.22360146045684814, 0.21901917457580566, 0.218156635761261]}, {"step": 3400, "epoch": 7, "train_loss": 0.1063101589679718, "cos_r_negg": [0.07290801405906677, 0.09812893718481064, 0.11037027835845947], "cos_innovation_negg": [0.5527506470680237, 0.6091886758804321, 0.7597736716270447], "cos_apical_negg": [0.07290801405906677, 0.09812894463539124, 0.11037027835845947], "cos_Ac_negg": [0.5524070858955383, 0.6110753417015076, 0.7611789703369141], "r_norm": [0.2745347023010254, 0.2698018550872803, 0.2688489556312561]}, {"step": 3500, "epoch": 7, "train_loss": 0.037357836961746216, "cos_r_negg": [0.06743920594453812, 0.08692482113838196, 0.13602200150489807], "cos_innovation_negg": [0.5601925849914551, 0.604422926902771, 0.7443182468414307], "cos_apical_negg": [0.06743919849395752, 0.08692482113838196, 0.13602200150489807], "cos_Ac_negg": [0.5611312985420227, 0.6055264472961426, 0.7438371181488037], "r_norm": [0.2416018843650818, 0.23779501020908356, 0.23534145951271057], "ldr": {"dL_sdil": -0.006105475127696991, "dL_bp": -0.009596720337867737, "ratio": 0.6362043398936377}}, {"step": 3600, "epoch": 7, "train_loss": 0.029826544225215912, "cos_r_negg": [0.08168534189462662, 0.10181734710931778, 0.1450318992137909], "cos_innovation_negg": [0.5346965789794922, 0.6047930717468262, 0.7627032399177551], "cos_apical_negg": [0.08168533444404602, 0.10181735455989838, 0.1450318992137909], "cos_Ac_negg": [0.5337558388710022, 0.6045243144035339, 0.7588868141174316], "r_norm": [0.26648473739624023, 0.26119711995124817, 0.2588616609573364]}, {"step": 3700, "epoch": 7, "train_loss": 0.031768057495355606, "cos_r_negg": [0.08394169807434082, 0.10271388292312622, 0.14723770320415497], "cos_innovation_negg": [0.5592381954193115, 0.609021008014679, 0.7928593158721924], "cos_apical_negg": [0.08394169807434082, 0.10271389037370682, 0.14723770320415497], "cos_Ac_negg": [0.5604240894317627, 0.6106314659118652, 0.7925615906715393], "r_norm": [0.2781471908092499, 0.2734353542327881, 0.27141472697257996]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9639, "test_loss": 0.13228435287475587}, {"step": 3800, "epoch": 8, "train_loss": 0.05007845163345337, "cos_r_negg": [0.0717189684510231, 0.08457625657320023, 0.10764709860086441], "cos_innovation_negg": [0.5546162128448486, 0.5975524187088013, 0.7884159088134766], "cos_apical_negg": [0.0717189610004425, 0.08457625657320023, 0.10764710605144501], "cos_Ac_negg": [0.5563030242919922, 0.6000141501426697, 0.789344310760498], "r_norm": [0.24482181668281555, 0.24063773453235626, 0.23882639408111572]}, {"step": 3900, "epoch": 8, "train_loss": 0.08228941261768341, "cos_r_negg": [0.06533907353878021, 0.07992380857467651, 0.147702157497406], "cos_innovation_negg": [0.5418471097946167, 0.5887052416801453, 0.7823539972305298], "cos_apical_negg": [0.06533907353878021, 0.07992380857467651, 0.147702157497406], "cos_Ac_negg": [0.5450972318649292, 0.5908803939819336, 0.7821483612060547], "r_norm": [0.25028395652770996, 0.24401697516441345, 0.24258068203926086]}, {"step": 4000, "epoch": 8, "train_loss": 0.05164622142910957, "cos_r_negg": [0.06902293860912323, 0.08827603608369827, 0.1320556104183197], "cos_innovation_negg": [0.5270317196846008, 0.5888156294822693, 0.7667114734649658], "cos_apical_negg": [0.06902293115854263, 0.08827603608369827, 0.1320556104183197], "cos_Ac_negg": [0.5293920040130615, 0.591464638710022, 0.7683452367782593], "r_norm": [0.2521548867225647, 0.2463391125202179, 0.24513624608516693], "ldr": {"dL_sdil": -0.009338408708572388, "dL_bp": -0.015852496027946472, "ratio": 0.5890812836104575}}, {"step": 4100, "epoch": 8, "train_loss": 0.02004571259021759, "cos_r_negg": [0.0625336766242981, 0.08250875771045685, 0.1356448531150818], "cos_innovation_negg": [0.5627686977386475, 0.5938019752502441, 0.7631560564041138], "cos_apical_negg": [0.0625336766242981, 0.08250876516103745, 0.1356448382139206], "cos_Ac_negg": [0.5648341178894043, 0.5962578058242798, 0.7650654315948486], "r_norm": [0.24856719374656677, 0.24405287206172943, 0.24061977863311768]}, {"step": 4200, "epoch": 8, "train_loss": 0.09874685108661652, "cos_r_negg": [0.07091013342142105, 0.07042770832777023, 0.13188888132572174], "cos_innovation_negg": [0.5426219701766968, 0.6038415431976318, 0.780397355556488], "cos_apical_negg": [0.07091013342142105, 0.07042770832777023, 0.13188888132572174], "cos_Ac_negg": [0.5441992282867432, 0.6068688631057739, 0.78309166431427], "r_norm": [0.2244134247303009, 0.2188449203968048, 0.21817874908447266]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9655, "test_loss": 0.13557769584655763}, {"step": 4300, "epoch": 9, "train_loss": 0.02034987322986126, "cos_r_negg": [0.06329119205474854, 0.07320599257946014, 0.10350732505321503], "cos_innovation_negg": [0.5358413457870483, 0.593376636505127, 0.7736502885818481], "cos_apical_negg": [0.06329119205474854, 0.07320599257946014, 0.10350732505321503], "cos_Ac_negg": [0.5360220074653625, 0.5959383845329285, 0.7705438137054443], "r_norm": [0.2687934637069702, 0.26312386989593506, 0.26151275634765625]}, {"step": 4400, "epoch": 9, "train_loss": 0.01585252955555916, "cos_r_negg": [0.0704190656542778, 0.08715783804655075, 0.12539517879486084], "cos_innovation_negg": [0.5446029901504517, 0.602871298789978, 0.7813178896903992], "cos_apical_negg": [0.0704190656542778, 0.08715783804655075, 0.12539517879486084], "cos_Ac_negg": [0.5481078624725342, 0.6076574325561523, 0.78309166431427], "r_norm": [0.23737162351608276, 0.2324787676334381, 0.22979232668876648]}, {"step": 4500, "epoch": 9, "train_loss": 0.19144883751869202, "cos_r_negg": [0.05317222326993942, 0.06681753695011139, 0.14209970831871033], "cos_innovation_negg": [0.5579216480255127, 0.6123360395431519, 0.7830038070678711], "cos_apical_negg": [0.05317222326993942, 0.06681753695011139, 0.14209970831871033], "cos_Ac_negg": [0.560858964920044, 0.6170240044593811, 0.7845879793167114], "r_norm": [0.22783441841602325, 0.2232532948255539, 0.2201940417289734], "ldr": {"dL_sdil": -0.007011055946350098, "dL_bp": -0.010684967041015625, "ratio": 0.656160746162085}}, {"step": 4600, "epoch": 9, "train_loss": 0.03670297563076019, "cos_r_negg": [0.0664428174495697, 0.07982797175645828, 0.13727284967899323], "cos_innovation_negg": [0.5728162527084351, 0.62354975938797, 0.7998867034912109], "cos_apical_negg": [0.0664428174495697, 0.07982797175645828, 0.13727283477783203], "cos_Ac_negg": [0.5757042169570923, 0.6258006691932678, 0.8005550503730774], "r_norm": [0.283415824174881, 0.2762840688228607, 0.2732480466365814]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9698, "test_loss": 0.1227607042312622}, {"step": 4700, "epoch": 10, "train_loss": 0.04839279130101204, "cos_r_negg": [0.06364335119724274, 0.07567989826202393, 0.1274554431438446], "cos_innovation_negg": [0.5271023511886597, 0.5936386585235596, 0.7894782423973083], "cos_apical_negg": [0.06364335119724274, 0.07567991316318512, 0.1274554431438446], "cos_Ac_negg": [0.532446026802063, 0.5963029861450195, 0.7881503701210022], "r_norm": [0.2292705476284027, 0.22390416264533997, 0.2221139371395111]}, {"step": 4800, "epoch": 10, "train_loss": 0.03952376917004585, "cos_r_negg": [0.046887755393981934, 0.06370607018470764, 0.12445493787527084], "cos_innovation_negg": [0.5435724854469299, 0.6021091341972351, 0.797446072101593], "cos_apical_negg": [0.046887755393981934, 0.06370607018470764, 0.12445493787527084], "cos_Ac_negg": [0.5482916235923767, 0.6075999736785889, 0.7988876104354858], "r_norm": [0.17225481569766998, 0.16954630613327026, 0.16898219287395477]}, {"step": 4900, "epoch": 10, "train_loss": 0.03187403082847595, "cos_r_negg": [0.0643419474363327, 0.08255770057439804, 0.13570451736450195], "cos_innovation_negg": [0.5599974393844604, 0.6049967408180237, 0.7847260236740112], "cos_apical_negg": [0.0643419474363327, 0.08255770057439804, 0.13570451736450195], "cos_Ac_negg": [0.5625724792480469, 0.6064321994781494, 0.7859264016151428], "r_norm": [0.23878632485866547, 0.2340824455022812, 0.23296594619750977]}, {"step": 5000, "epoch": 10, "train_loss": 0.02667912095785141, "cos_r_negg": [0.06717013567686081, 0.06284972280263901, 0.1056877002120018], "cos_innovation_negg": [0.552981436252594, 0.6072264909744263, 0.7859450578689575], "cos_apical_negg": [0.06717013567686081, 0.06284972280263901, 0.1056877002120018], "cos_Ac_negg": [0.557763934135437, 0.6128823757171631, 0.788381814956665], "r_norm": [0.2379167377948761, 0.2322022020816803, 0.23069347441196442], "ldr": {"dL_sdil": -0.00891779363155365, "dL_bp": -0.015383400022983551, "ratio": 0.5797023816731042}}, {"step": 5100, "epoch": 10, "train_loss": 0.0677889883518219, "cos_r_negg": [0.04698483645915985, 0.07755410671234131, 0.1409112960100174], "cos_innovation_negg": [0.5482721924781799, 0.620044469833374, 0.7865081429481506], "cos_apical_negg": [0.04698483645915985, 0.07755410671234131, 0.1409112960100174], "cos_Ac_negg": [0.5589918494224548, 0.6287689805030823, 0.7922115325927734], "r_norm": [0.2318282574415207, 0.22801828384399414, 0.22497932612895966]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9652, "test_loss": 0.13374782333374025}, {"step": 5200, "epoch": 11, "train_loss": 0.0038604428991675377, "cos_r_negg": [0.058167487382888794, 0.07114555686712265, 0.11589882522821426], "cos_innovation_negg": [0.5481350421905518, 0.6034847497940063, 0.7792742848396301], "cos_apical_negg": [0.058167487382888794, 0.07114555686712265, 0.11589881777763367], "cos_Ac_negg": [0.5543985366821289, 0.6114673614501953, 0.7864778637886047], "r_norm": [0.22882503271102905, 0.2241784632205963, 0.22140666842460632]}, {"step": 5300, "epoch": 11, "train_loss": 0.07033146917819977, "cos_r_negg": [0.04950704425573349, 0.0705556720495224, 0.10611505806446075], "cos_innovation_negg": [0.5287195444107056, 0.5920501351356506, 0.7762441635131836], "cos_apical_negg": [0.04950704053044319, 0.0705556720495224, 0.10611505806446075], "cos_Ac_negg": [0.5348626375198364, 0.5972569584846497, 0.7768690586090088], "r_norm": [0.23408210277557373, 0.23052264750003815, 0.2266005277633667]}, {"step": 5400, "epoch": 11, "train_loss": 0.08506366610527039, "cos_r_negg": [0.05567062646150589, 0.07443711906671524, 0.1290496587753296], "cos_innovation_negg": [0.5281569957733154, 0.5864236354827881, 0.7631567120552063], "cos_apical_negg": [0.05567063018679619, 0.07443712651729584, 0.1290496587753296], "cos_Ac_negg": [0.5394855737686157, 0.5949428081512451, 0.7680245041847229], "r_norm": [0.2382836788892746, 0.23389361798763275, 0.2303137481212616]}, {"step": 5500, "epoch": 11, "train_loss": 0.012573130428791046, "cos_r_negg": [0.06361153721809387, 0.06807530671358109, 0.12357844412326813], "cos_innovation_negg": [0.5378892421722412, 0.6000767350196838, 0.7792697548866272], "cos_apical_negg": [0.06361154466867447, 0.06807530671358109, 0.12357844412326813], "cos_Ac_negg": [0.5450212955474854, 0.6080093383789062, 0.7858061194419861], "r_norm": [0.21391001343727112, 0.20936641097068787, 0.20761336386203766], "ldr": {"dL_sdil": -0.006099686026573181, "dL_bp": -0.011147961020469666, "ratio": 0.5471571003318956}}, {"step": 5600, "epoch": 11, "train_loss": 0.009609967470169067, "cos_r_negg": [0.06594008207321167, 0.06625820696353912, 0.07251353561878204], "cos_innovation_negg": [0.527391791343689, 0.5948604345321655, 0.7574151754379272], "cos_apical_negg": [0.06594006717205048, 0.06625820696353912, 0.07251353561878204], "cos_Ac_negg": [0.5348454713821411, 0.6017602682113647, 0.7670135498046875], "r_norm": [0.22620606422424316, 0.2204313576221466, 0.21838465332984924]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9696, "test_loss": 0.12623830223083496}, {"step": 5700, "epoch": 12, "train_loss": 0.016506992280483246, "cos_r_negg": [0.0547497496008873, 0.07152171432971954, 0.09657566994428635], "cos_innovation_negg": [0.5452012419700623, 0.5976607799530029, 0.7862253785133362], "cos_apical_negg": [0.0547497496008873, 0.07152171432971954, 0.09657566994428635], "cos_Ac_negg": [0.5535536408424377, 0.6069254875183105, 0.7960631847381592], "r_norm": [0.24224236607551575, 0.23622053861618042, 0.23375500738620758]}, {"step": 5800, "epoch": 12, "train_loss": 0.0026011494919657707, "cos_r_negg": [0.05949844419956207, 0.06380505114793777, 0.11441205441951752], "cos_innovation_negg": [0.5408796668052673, 0.5976915955543518, 0.7925673723220825], "cos_apical_negg": [0.05949845165014267, 0.06380504369735718, 0.11441206932067871], "cos_Ac_negg": [0.5505724549293518, 0.6095927953720093, 0.8027863502502441], "r_norm": [0.2224837690591812, 0.217766672372818, 0.2157045602798462]}, {"step": 5900, "epoch": 12, "train_loss": 0.021796507760882378, "cos_r_negg": [0.054697148501873016, 0.07463476061820984, 0.0868355929851532], "cos_innovation_negg": [0.5256735682487488, 0.5992733240127563, 0.7848494052886963], "cos_apical_negg": [0.054697152227163315, 0.07463476061820984, 0.0868355929851532], "cos_Ac_negg": [0.5358738899230957, 0.6056530475616455, 0.7925519943237305], "r_norm": [0.21211518347263336, 0.2071298062801361, 0.2064683437347412]}, {"step": 6000, "epoch": 12, "train_loss": 0.026512151584029198, "cos_r_negg": [0.057589564472436905, 0.06602048873901367, 0.07767926156520844], "cos_innovation_negg": [0.5288076400756836, 0.6061176061630249, 0.7742867469787598], "cos_apical_negg": [0.0575895681977272, 0.06602048873901367, 0.07767926156520844], "cos_Ac_negg": [0.5394042730331421, 0.6129321455955505, 0.7777043581008911], "r_norm": [0.21243371069431305, 0.20848143100738525, 0.20531213283538818], "ldr": {"dL_sdil": -0.009594418108463287, "dL_bp": -0.020150206983089447, "ratio": 0.4761448910433109}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9672, "test_loss": 0.13024798736572266}, {"step": 6100, "epoch": 13, "train_loss": 0.009351560845971107, "cos_r_negg": [0.052101340144872665, 0.058585524559020996, 0.06036121025681496], "cos_innovation_negg": [0.5419372320175171, 0.6053557395935059, 0.783679187297821], "cos_apical_negg": [0.052101343870162964, 0.058585528284311295, 0.06036120653152466], "cos_Ac_negg": [0.5503400564193726, 0.6160815954208374, 0.7936914563179016], "r_norm": [0.21938712894916534, 0.21519601345062256, 0.2139711081981659]}, {"step": 6200, "epoch": 13, "train_loss": 0.006052996497601271, "cos_r_negg": [0.053515542298555374, 0.055314261466264725, 0.05995969474315643], "cos_innovation_negg": [0.5447695255279541, 0.6130589842796326, 0.7757765054702759], "cos_apical_negg": [0.053515542298555374, 0.055314261466264725, 0.05995970219373703], "cos_Ac_negg": [0.5529961585998535, 0.6255960464477539, 0.7874360084533691], "r_norm": [0.20926839113235474, 0.20431242883205414, 0.20193041861057281]}, {"step": 6300, "epoch": 13, "train_loss": 0.018606022000312805, "cos_r_negg": [0.05628553405404091, 0.05468010902404785, 0.057541463524103165], "cos_innovation_negg": [0.5407936573028564, 0.5999512672424316, 0.7780349254608154], "cos_apical_negg": [0.05628553777933121, 0.05468010902404785, 0.057541463524103165], "cos_Ac_negg": [0.550012469291687, 0.6131508350372314, 0.7917049527168274], "r_norm": [0.2285028100013733, 0.22168998420238495, 0.22110216319561005]}, {"step": 6400, "epoch": 13, "train_loss": 0.004905556328594685, "cos_r_negg": [0.050309933722019196, 0.06324630230665207, 0.08564186096191406], "cos_innovation_negg": [0.5209953784942627, 0.5941855907440186, 0.7652787566184998], "cos_apical_negg": [0.0503099262714386, 0.06324630230665207, 0.08564185351133347], "cos_Ac_negg": [0.5333911180496216, 0.6052359342575073, 0.771978497505188], "r_norm": [0.2235257625579834, 0.2172807902097702, 0.21643313765525818]}, {"step": 6500, "epoch": 13, "train_loss": 0.0013855393044650555, "cos_r_negg": [0.05341285839676857, 0.06854744255542755, 0.1042645275592804], "cos_innovation_negg": [0.5446828007698059, 0.6036902666091919, 0.7699993848800659], "cos_apical_negg": [0.05341285467147827, 0.06854744255542755, 0.10426453500986099], "cos_Ac_negg": [0.5551351308822632, 0.6163210272789001, 0.7806403636932373], "r_norm": [0.22807006537914276, 0.2236315906047821, 0.22066211700439453], "ldr": {"dL_sdil": -0.008510112762451172, "dL_bp": -0.016563981771469116, "ratio": 0.513772164197231}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9695, "test_loss": 0.12776789474487305}, {"step": 6600, "epoch": 14, "train_loss": 0.0038456113543361425, "cos_r_negg": [0.05062942951917648, 0.07090011239051819, 0.1098744124174118], "cos_innovation_negg": [0.5379418730735779, 0.5994153022766113, 0.7756435871124268], "cos_apical_negg": [0.05062942951917648, 0.07090011984109879, 0.1098744198679924], "cos_Ac_negg": [0.5495676398277283, 0.6087256669998169, 0.7847073674201965], "r_norm": [0.20909591019153595, 0.20454992353916168, 0.2028166502714157]}, {"step": 6700, "epoch": 14, "train_loss": 0.006470425985753536, "cos_r_negg": [0.05011637881398201, 0.05745463818311691, 0.10158899426460266], "cos_innovation_negg": [0.524763286113739, 0.5984794497489929, 0.7564308047294617], "cos_apical_negg": [0.05011637881398201, 0.05745463818311691, 0.10158898681402206], "cos_Ac_negg": [0.5375996828079224, 0.6064271926879883, 0.7629046440124512], "r_norm": [0.2094632089138031, 0.20427276194095612, 0.2030295431613922]}, {"step": 6800, "epoch": 14, "train_loss": 0.02861160971224308, "cos_r_negg": [0.0517265610396862, 0.062266431748867035, 0.06925496459007263], "cos_innovation_negg": [0.5419613122940063, 0.6001811027526855, 0.7763111591339111], "cos_apical_negg": [0.0517265647649765, 0.06226643547415733, 0.06925496459007263], "cos_Ac_negg": [0.5514795780181885, 0.6109452843666077, 0.7812302112579346], "r_norm": [0.20821134746074677, 0.20343603193759918, 0.20183655619621277]}, {"step": 6900, "epoch": 14, "train_loss": 0.012429247610270977, "cos_r_negg": [0.05247191712260246, 0.060937635600566864, 0.06693011522293091], "cos_innovation_negg": [0.5486399531364441, 0.6003872156143188, 0.7809118032455444], "cos_apical_negg": [0.05247192084789276, 0.060937635600566864, 0.06693011522293091], "cos_Ac_negg": [0.5589513182640076, 0.6118178367614746, 0.7880287170410156], "r_norm": [0.212667778134346, 0.2083602100610733, 0.20657868683338165]}, {"step": 7000, "epoch": 14, "train_loss": 0.004722911864519119, "cos_r_negg": [0.0473896786570549, 0.06399355083703995, 0.06589870899915695], "cos_innovation_negg": [0.5357134938240051, 0.5969662666320801, 0.7830662727355957], "cos_apical_negg": [0.0473896823823452, 0.06399355083703995, 0.06589871644973755], "cos_Ac_negg": [0.5482646226882935, 0.6102514266967773, 0.7924281358718872], "r_norm": [0.20618706941604614, 0.20211279392242432, 0.2001674473285675], "ldr": {"dL_sdil": -0.00756506621837616, "dL_bp": -0.014824427664279938, "ratio": 0.5103108456999318}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.971, "test_loss": 0.13526744537353516}], "final": {"test_acc": 0.971, "test_loss": 0.13526744537353516, "wall_s": 25.47541618347168, "cos_r_negg": [0.04763709008693695, 0.05940413102507591, 0.05433851107954979], "cos_innovation_negg": [0.5352548360824585, 0.5923671722412109, 0.7811884880065918], "cos_apical_negg": [0.04763708636164665, 0.059404127299785614, 0.05433851107954979], "cos_Ac_negg": [0.5465336441993713, 0.6060895919799805, 0.7910354137420654]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_matched_s1.json b/results/nuis_ctrl_v1_rho0p05_matched_s1.json new file mode 100644 index 0000000..102f788 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_matched_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_matched_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.014000799506902695, -0.013633744791150093, -0.0066742198541760445], "cos_innovation_negg": [0.014459304511547089, -0.013818614184856415, -0.0063325027003884315], "cos_apical_negg": [0.01400079857558012, -0.013633744791150093, -0.006674220319837332], "cos_Ac_negg": [0.014459235593676567, -0.013818634673953056, -0.0063325269147753716], "r_norm": [4.738112449645996, 4.797549247741699, 4.787727355957031], "ldr": {"dL_sdil": -0.042402029037475586, "dL_bp": -0.21435141563415527, "ratio": 0.1978154840360156}}, {"step": 100, "epoch": 0, "train_loss": 0.2781984210014343, "cos_r_negg": [0.26685237884521484, 0.21982751786708832, 0.22992366552352905], "cos_innovation_negg": [0.5157362222671509, 0.41939112544059753, 0.3292425274848938], "cos_apical_negg": [0.26685237884521484, 0.21982751786708832, 0.22992366552352905], "cos_Ac_negg": [0.5156465172767639, 0.4194236695766449, 0.3292240500450134], "r_norm": [1.0385547876358032, 1.0394731760025024, 1.056185007095337]}, {"step": 200, "epoch": 0, "train_loss": 0.3677998185157776, "cos_r_negg": [0.263663113117218, 0.2083938717842102, 0.22713875770568848], "cos_innovation_negg": [0.5560674071311951, 0.46555498242378235, 0.4336373507976532], "cos_apical_negg": [0.263663113117218, 0.2083938866853714, 0.22713875770568848], "cos_Ac_negg": [0.555426299571991, 0.46511727571487427, 0.4329182207584381], "r_norm": [0.8922678232192993, 0.8953602313995361, 0.9070045948028564]}, {"step": 300, "epoch": 0, "train_loss": 0.23251374065876007, "cos_r_negg": [0.23529763519763947, 0.21924659609794617, 0.2554989457130432], "cos_innovation_negg": [0.5555249452590942, 0.5305030941963196, 0.5329081416130066], "cos_apical_negg": [0.23529765009880066, 0.21924659609794617, 0.2554989457130432], "cos_Ac_negg": [0.5553916692733765, 0.5303660035133362, 0.5327898263931274], "r_norm": [0.6125134229660034, 0.6164205074310303, 0.6273356676101685]}, {"step": 400, "epoch": 0, "train_loss": 0.15909647941589355, "cos_r_negg": [0.22536510229110718, 0.22064456343650818, 0.24368901550769806], "cos_innovation_negg": [0.5665348768234253, 0.5570060610771179, 0.5425494909286499], "cos_apical_negg": [0.22536510229110718, 0.22064456343650818, 0.24368901550769806], "cos_Ac_negg": [0.5663563013076782, 0.5569068789482117, 0.542511522769928], "r_norm": [0.6293011903762817, 0.632056474685669, 0.6434175968170166]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9403, "test_loss": 0.20267537536621094}, {"step": 500, "epoch": 1, "train_loss": 0.18902839720249176, "cos_r_negg": [0.1790933609008789, 0.20257514715194702, 0.23208177089691162], "cos_innovation_negg": [0.5719155073165894, 0.5642278790473938, 0.5872927904129028], "cos_apical_negg": [0.1790933609008789, 0.20257514715194702, 0.23208177089691162], "cos_Ac_negg": [0.5718954801559448, 0.5642244815826416, 0.5871847867965698], "r_norm": [0.5600966811180115, 0.5626634359359741, 0.5669237971305847], "ldr": {"dL_sdil": -0.011338025331497192, "dL_bp": -0.01136857271194458, "ratio": 0.9973129977507825}}, {"step": 600, "epoch": 1, "train_loss": 0.11155551671981812, "cos_r_negg": [0.20599210262298584, 0.194766104221344, 0.20247206091880798], "cos_innovation_negg": [0.5785467624664307, 0.5494545698165894, 0.6112366914749146], "cos_apical_negg": [0.20599213242530823, 0.194766104221344, 0.20247206091880798], "cos_Ac_negg": [0.5783642530441284, 0.5492405891418457, 0.6110966205596924], "r_norm": [0.5204837322235107, 0.5207526683807373, 0.5278429985046387]}, {"step": 700, "epoch": 1, "train_loss": 0.19533555209636688, "cos_r_negg": [0.17774969339370728, 0.19593371450901031, 0.21875163912773132], "cos_innovation_negg": [0.6004726886749268, 0.5840119123458862, 0.6007018089294434], "cos_apical_negg": [0.17774969339370728, 0.1959337294101715, 0.21875163912773132], "cos_Ac_negg": [0.6002299189567566, 0.5839093923568726, 0.6005539894104004], "r_norm": [0.4677206873893738, 0.47016650438308716, 0.4738680422306061]}, {"step": 800, "epoch": 1, "train_loss": 0.15825039148330688, "cos_r_negg": [0.16074857115745544, 0.1626344472169876, 0.1874714493751526], "cos_innovation_negg": [0.5607525110244751, 0.5336446166038513, 0.5835872888565063], "cos_apical_negg": [0.16074855625629425, 0.1626344621181488, 0.1874714493751526], "cos_Ac_negg": [0.5605154037475586, 0.5334181785583496, 0.5833826065063477], "r_norm": [0.4187534749507904, 0.42067137360572815, 0.42354151606559753]}, {"step": 900, "epoch": 1, "train_loss": 0.18467900156974792, "cos_r_negg": [0.20914487540721893, 0.22298023104667664, 0.2523660659790039], "cos_innovation_negg": [0.5801753997802734, 0.562985897064209, 0.606001615524292], "cos_apical_negg": [0.20914486050605774, 0.22298023104667664, 0.2523660957813263], "cos_Ac_negg": [0.5800694227218628, 0.5628949403762817, 0.6058626174926758], "r_norm": [0.5190070867538452, 0.5219466686248779, 0.5291485786437988]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9506, "test_loss": 0.17010464248657226}, {"step": 1000, "epoch": 2, "train_loss": 0.04992591589689255, "cos_r_negg": [0.19572226703166962, 0.17794883251190186, 0.22146213054656982], "cos_innovation_negg": [0.5998021960258484, 0.5672405958175659, 0.6154801845550537], "cos_apical_negg": [0.19572226703166962, 0.17794883251190186, 0.22146213054656982], "cos_Ac_negg": [0.5996326804161072, 0.5670913457870483, 0.6152791976928711], "r_norm": [0.4730270802974701, 0.470811665058136, 0.4747304916381836], "ldr": {"dL_sdil": -0.015341967344284058, "dL_bp": -0.0163649320602417, "ratio": 0.9374904391786071}}, {"step": 1100, "epoch": 2, "train_loss": 0.20945867896080017, "cos_r_negg": [0.1694614589214325, 0.18095383048057556, 0.2116498351097107], "cos_innovation_negg": [0.5619527697563171, 0.5650385618209839, 0.6543017625808716], "cos_apical_negg": [0.1694614589214325, 0.18095383048057556, 0.2116498053073883], "cos_Ac_negg": [0.5618287324905396, 0.56494140625, 0.654178261756897], "r_norm": [0.4545907974243164, 0.4580495357513428, 0.4582388401031494]}, {"step": 1200, "epoch": 2, "train_loss": 0.1839890480041504, "cos_r_negg": [0.1600516140460968, 0.17260783910751343, 0.1995244026184082], "cos_innovation_negg": [0.5563066005706787, 0.5602741241455078, 0.6617388129234314], "cos_apical_negg": [0.1600516140460968, 0.17260785400867462, 0.1995244026184082], "cos_Ac_negg": [0.5561938881874084, 0.5601537227630615, 0.661475658416748], "r_norm": [0.40716421604156494, 0.4083688259124756, 0.4146285057067871]}, {"step": 1300, "epoch": 2, "train_loss": 0.15830054879188538, "cos_r_negg": [0.15883848071098328, 0.1917852759361267, 0.20369289815425873], "cos_innovation_negg": [0.555382490158081, 0.5663414597511292, 0.6880208849906921], "cos_apical_negg": [0.15883848071098328, 0.1917852759361267, 0.20369289815425873], "cos_Ac_negg": [0.5552250742912292, 0.5662208795547485, 0.6878046989440918], "r_norm": [0.43299487233161926, 0.4335848093032837, 0.4377847909927368]}, {"step": 1400, "epoch": 2, "train_loss": 0.19248934090137482, "cos_r_negg": [0.14802604913711548, 0.17009006440639496, 0.23110854625701904], "cos_innovation_negg": [0.559179425239563, 0.5863549113273621, 0.6957074403762817], "cos_apical_negg": [0.14802604913711548, 0.17009006440639496, 0.23110854625701904], "cos_Ac_negg": [0.5590461492538452, 0.586161196231842, 0.6953648924827576], "r_norm": [0.37540918588638306, 0.37148696184158325, 0.37575411796569824]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9596, "test_loss": 0.1394033809661865}, {"step": 1500, "epoch": 3, "train_loss": 0.08944995701313019, "cos_r_negg": [0.14514648914337158, 0.16608357429504395, 0.19010353088378906], "cos_innovation_negg": [0.5726072788238525, 0.5724003911018372, 0.7225419282913208], "cos_apical_negg": [0.14514648914337158, 0.16608357429504395, 0.19010353088378906], "cos_Ac_negg": [0.5723007321357727, 0.5721114873886108, 0.7221858501434326], "r_norm": [0.3024035096168518, 0.30209675431251526, 0.3078489899635315], "ldr": {"dL_sdil": -0.003830850124359131, "dL_bp": -0.004313953220844269, "ratio": 0.8880138305276775}}, {"step": 1600, "epoch": 3, "train_loss": 0.15920960903167725, "cos_r_negg": [0.1307712346315384, 0.1694299578666687, 0.19286535680294037], "cos_innovation_negg": [0.5470030307769775, 0.5656390190124512, 0.741963267326355], "cos_apical_negg": [0.1307712197303772, 0.1694299727678299, 0.19286534190177917], "cos_Ac_negg": [0.546863853931427, 0.565504789352417, 0.741804838180542], "r_norm": [0.37343645095825195, 0.3703274130821228, 0.3722206950187683]}, {"step": 1700, "epoch": 3, "train_loss": 0.10022221505641937, "cos_r_negg": [0.160159632563591, 0.15611574053764343, 0.1771431416273117], "cos_innovation_negg": [0.529837429523468, 0.5634852051734924, 0.6926331520080566], "cos_apical_negg": [0.160159632563591, 0.15611574053764343, 0.1771431565284729], "cos_Ac_negg": [0.5297232866287231, 0.5634281635284424, 0.6925235986709595], "r_norm": [0.37219926714897156, 0.36870720982551575, 0.36905521154403687]}, {"step": 1800, "epoch": 3, "train_loss": 0.0368051752448082, "cos_r_negg": [0.13545368611812592, 0.11821255087852478, 0.1634964942932129], "cos_innovation_negg": [0.5214605331420898, 0.5495294332504272, 0.6641377210617065], "cos_apical_negg": [0.13545368611812592, 0.11821255087852478, 0.1634964942932129], "cos_Ac_negg": [0.5212026238441467, 0.5492298603057861, 0.6633946895599365], "r_norm": [0.39454638957977295, 0.39656558632850647, 0.40040087699890137]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9582, "test_loss": 0.15445995407104493}, {"step": 1900, "epoch": 4, "train_loss": 0.06744775176048279, "cos_r_negg": [0.11355800181627274, 0.15126921236515045, 0.15317592024803162], "cos_innovation_negg": [0.5673056244850159, 0.5809839367866516, 0.7085917592048645], "cos_apical_negg": [0.11355799436569214, 0.15126919746398926, 0.15317592024803162], "cos_Ac_negg": [0.567139744758606, 0.5805246233940125, 0.7078638672828674], "r_norm": [0.3425997793674469, 0.3435506820678711, 0.3472229242324829]}, {"step": 2000, "epoch": 4, "train_loss": 0.133309468626976, "cos_r_negg": [0.0911845788359642, 0.11686769872903824, 0.12235550582408905], "cos_innovation_negg": [0.5420997142791748, 0.5717331767082214, 0.7403692007064819], "cos_apical_negg": [0.0911845788359642, 0.11686769872903824, 0.12235550582408905], "cos_Ac_negg": [0.5418422222137451, 0.5715316534042358, 0.740033745765686], "r_norm": [0.2965013384819031, 0.29893380403518677, 0.3011382818222046], "ldr": {"dL_sdil": -0.008002370595932007, "dL_bp": -0.009968265891075134, "ratio": 0.8027846250667081}}, {"step": 2100, "epoch": 4, "train_loss": 0.03637475147843361, "cos_r_negg": [0.11173398792743683, 0.13662609457969666, 0.1658332794904709], "cos_innovation_negg": [0.5483249425888062, 0.5740309357643127, 0.7501567006111145], "cos_apical_negg": [0.11173398792743683, 0.13662609457969666, 0.1658332794904709], "cos_Ac_negg": [0.5482739806175232, 0.5739814043045044, 0.7498853206634521], "r_norm": [0.2998882234096527, 0.2994261384010315, 0.3042312264442444]}, {"step": 2200, "epoch": 4, "train_loss": 0.022333752363920212, "cos_r_negg": [0.10447806119918823, 0.1357472538948059, 0.17529860138893127], "cos_innovation_negg": [0.5510979890823364, 0.5893969535827637, 0.7025282382965088], "cos_apical_negg": [0.10447806119918823, 0.1357472538948059, 0.17529860138893127], "cos_Ac_negg": [0.55082768201828, 0.5890421867370605, 0.7021288871765137], "r_norm": [0.33024102449417114, 0.3275192081928253, 0.3374108374118805]}, {"step": 2300, "epoch": 4, "train_loss": 0.18817636370658875, "cos_r_negg": [0.10440659523010254, 0.139730766415596, 0.15772543847560883], "cos_innovation_negg": [0.5324651002883911, 0.5545135736465454, 0.7314645648002625], "cos_apical_negg": [0.10440659523010254, 0.139730766415596, 0.15772542357444763], "cos_Ac_negg": [0.5320799350738525, 0.5541362762451172, 0.7308579683303833], "r_norm": [0.3156967759132385, 0.31805694103240967, 0.3205592930316925]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9652, "test_loss": 0.1349000961303711}, {"step": 2400, "epoch": 5, "train_loss": 0.13673873245716095, "cos_r_negg": [0.06086712330579758, 0.08893437683582306, 0.138218492269516], "cos_innovation_negg": [0.5715898275375366, 0.5935612320899963, 0.7480977773666382], "cos_apical_negg": [0.060867127031087875, 0.08893437683582306, 0.13821850717067719], "cos_Ac_negg": [0.5715224742889404, 0.5934250354766846, 0.74757319688797], "r_norm": [0.2423560917377472, 0.23958319425582886, 0.2430085837841034]}, {"step": 2500, "epoch": 5, "train_loss": 0.08846108615398407, "cos_r_negg": [0.06706833094358444, 0.10987864434719086, 0.15216797590255737], "cos_innovation_negg": [0.5385262966156006, 0.58486407995224, 0.7296252846717834], "cos_apical_negg": [0.06706832349300385, 0.10987864434719086, 0.15216797590255737], "cos_Ac_negg": [0.5377368330955505, 0.5837572813034058, 0.7292695045471191], "r_norm": [0.27576175332069397, 0.27765315771102905, 0.28112727403640747], "ldr": {"dL_sdil": -0.00995662808418274, "dL_bp": -0.013622581958770752, "ratio": 0.7308914062192353}}, {"step": 2600, "epoch": 5, "train_loss": 0.10714676976203918, "cos_r_negg": [0.11678977310657501, 0.12979930639266968, 0.18900683522224426], "cos_innovation_negg": [0.5647395849227905, 0.5982375144958496, 0.7303290367126465], "cos_apical_negg": [0.11678977310657501, 0.12979930639266968, 0.18900682032108307], "cos_Ac_negg": [0.564498782157898, 0.5980241298675537, 0.7301104068756104], "r_norm": [0.3022003769874573, 0.29887285828590393, 0.3037441074848175]}, {"step": 2700, "epoch": 5, "train_loss": 0.12422903627157211, "cos_r_negg": [0.0964074656367302, 0.1067756861448288, 0.15795418620109558], "cos_innovation_negg": [0.5652424097061157, 0.5902745723724365, 0.7420969605445862], "cos_apical_negg": [0.0964074656367302, 0.1067756861448288, 0.15795418620109558], "cos_Ac_negg": [0.5650827884674072, 0.5900545120239258, 0.7416421175003052], "r_norm": [0.28233474493026733, 0.2806481719017029, 0.2845459580421448]}, {"step": 2800, "epoch": 5, "train_loss": 0.19038572907447815, "cos_r_negg": [0.09882406890392303, 0.11376520991325378, 0.16026218235492706], "cos_innovation_negg": [0.5563746690750122, 0.568358302116394, 0.7109635472297668], "cos_apical_negg": [0.09882406890392303, 0.11376521736383438, 0.16026219725608826], "cos_Ac_negg": [0.5561993718147278, 0.5675990581512451, 0.7105340957641602], "r_norm": [0.30161112546920776, 0.29919740557670593, 0.30328208208084106]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.964, "test_loss": 0.12621053848266603}, {"step": 2900, "epoch": 6, "train_loss": 0.02114187926054001, "cos_r_negg": [0.07249077409505844, 0.09168776869773865, 0.15117180347442627], "cos_innovation_negg": [0.5390970706939697, 0.5703646540641785, 0.7006702423095703], "cos_apical_negg": [0.07249076664447784, 0.09168777614831924, 0.15117178857326508], "cos_Ac_negg": [0.5382399559020996, 0.5694425702095032, 0.6996619701385498], "r_norm": [0.2647293508052826, 0.2658502459526062, 0.2682751715183258]}, {"step": 3000, "epoch": 6, "train_loss": 0.20470091700553894, "cos_r_negg": [0.07848583906888962, 0.08388024568557739, 0.15430918335914612], "cos_innovation_negg": [0.5579773783683777, 0.5941951274871826, 0.7172616720199585], "cos_apical_negg": [0.07848583906888962, 0.08388024568557739, 0.15430918335914612], "cos_Ac_negg": [0.5577477216720581, 0.5935335755348206, 0.7163886427879333], "r_norm": [0.2652837336063385, 0.26477673649787903, 0.2703627049922943], "ldr": {"dL_sdil": -0.00884576141834259, "dL_bp": -0.011587589979171753, "ratio": 0.7633823283566735}}, {"step": 3100, "epoch": 6, "train_loss": 0.09332571923732758, "cos_r_negg": [0.08043253421783447, 0.10789922624826431, 0.15035396814346313], "cos_innovation_negg": [0.5523258447647095, 0.589907705783844, 0.7223434448242188], "cos_apical_negg": [0.08043253421783447, 0.10789922624826431, 0.15035396814346313], "cos_Ac_negg": [0.5521047711372375, 0.5893389582633972, 0.7221006155014038], "r_norm": [0.26571565866470337, 0.264682412147522, 0.26524415612220764]}, {"step": 3200, "epoch": 6, "train_loss": 0.08728523552417755, "cos_r_negg": [0.0591692179441452, 0.07672376185655594, 0.10397675633430481], "cos_innovation_negg": [0.5728427767753601, 0.5943782329559326, 0.7174142599105835], "cos_apical_negg": [0.059169214218854904, 0.07672376930713654, 0.10397675633430481], "cos_Ac_negg": [0.5727130770683289, 0.5937618613243103, 0.7167554497718811], "r_norm": [0.2354109287261963, 0.23463121056556702, 0.23798078298568726]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9652, "test_loss": 0.1265283432006836}, {"step": 3300, "epoch": 7, "train_loss": 0.049240585416555405, "cos_r_negg": [0.07296295464038849, 0.08288723230361938, 0.11177732050418854], "cos_innovation_negg": [0.552432656288147, 0.5862433314323425, 0.7291265726089478], "cos_apical_negg": [0.07296295464038849, 0.08288723975419998, 0.11177733540534973], "cos_Ac_negg": [0.5523402690887451, 0.5862059593200684, 0.7294195294380188], "r_norm": [0.24863889813423157, 0.2463444322347641, 0.25142961740493774]}, {"step": 3400, "epoch": 7, "train_loss": 0.036376114934682846, "cos_r_negg": [0.06327491998672485, 0.08526453375816345, 0.13728328049182892], "cos_innovation_negg": [0.5689361095428467, 0.5867960453033447, 0.7374023199081421], "cos_apical_negg": [0.06327491998672485, 0.08526453375816345, 0.13728328049182892], "cos_Ac_negg": [0.5688714981079102, 0.5868169069290161, 0.7381430864334106], "r_norm": [0.2511857748031616, 0.25147727131843567, 0.25480884313583374]}, {"step": 3500, "epoch": 7, "train_loss": 0.04425770416855812, "cos_r_negg": [0.07048217952251434, 0.09557612240314484, 0.11822102963924408], "cos_innovation_negg": [0.5470721125602722, 0.5822858810424805, 0.7139791250228882], "cos_apical_negg": [0.07048217952251434, 0.09557612240314484, 0.11822102963924408], "cos_Ac_negg": [0.5478157997131348, 0.5823948383331299, 0.7132679224014282], "r_norm": [0.21853691339492798, 0.21888524293899536, 0.2214718759059906], "ldr": {"dL_sdil": -0.00374777615070343, "dL_bp": -0.0060371458530426025, "ratio": 0.6207860869908626}}, {"step": 3600, "epoch": 7, "train_loss": 0.016083816066384315, "cos_r_negg": [0.07472144067287445, 0.0981871634721756, 0.14003993570804596], "cos_innovation_negg": [0.5394066572189331, 0.5919173359870911, 0.7581023573875427], "cos_apical_negg": [0.07472144067287445, 0.0981871634721756, 0.14003993570804596], "cos_Ac_negg": [0.5396754145622253, 0.5916072726249695, 0.7576492428779602], "r_norm": [0.27651557326316833, 0.27698010206222534, 0.2793763279914856]}, {"step": 3700, "epoch": 7, "train_loss": 0.02318321354687214, "cos_r_negg": [0.0632040798664093, 0.10064822435379028, 0.10517960786819458], "cos_innovation_negg": [0.5654623508453369, 0.5972422361373901, 0.7645934820175171], "cos_apical_negg": [0.0632040798664093, 0.10064822435379028, 0.10517960786819458], "cos_Ac_negg": [0.5651528835296631, 0.5971337556838989, 0.7645400762557983], "r_norm": [0.26289740204811096, 0.2633611559867859, 0.26702022552490234]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.966, "test_loss": 0.13126993370056153}, {"step": 3800, "epoch": 8, "train_loss": 0.033086568117141724, "cos_r_negg": [0.06775660812854767, 0.09610474109649658, 0.13352809846401215], "cos_innovation_negg": [0.5622141361236572, 0.5975745916366577, 0.7608423829078674], "cos_apical_negg": [0.06775660812854767, 0.09610474854707718, 0.13352811336517334], "cos_Ac_negg": [0.5616971254348755, 0.5979364514350891, 0.7606920003890991], "r_norm": [0.2271111011505127, 0.22552378475666046, 0.22792571783065796]}, {"step": 3900, "epoch": 8, "train_loss": 0.21299625933170319, "cos_r_negg": [0.070110023021698, 0.10455170273780823, 0.16838666796684265], "cos_innovation_negg": [0.549162745475769, 0.5947030186653137, 0.7463819980621338], "cos_apical_negg": [0.070110023021698, 0.10455170273780823, 0.16838666796684265], "cos_Ac_negg": [0.5506429076194763, 0.5961032509803772, 0.7443358898162842], "r_norm": [0.26119065284729004, 0.2603268623352051, 0.26267847418785095]}, {"step": 4000, "epoch": 8, "train_loss": 0.045660194009542465, "cos_r_negg": [0.0635378286242485, 0.061132095754146576, 0.09236867725849152], "cos_innovation_negg": [0.5339014530181885, 0.5708186030387878, 0.7236140966415405], "cos_apical_negg": [0.0635378286242485, 0.061132095754146576, 0.09236867725849152], "cos_Ac_negg": [0.5340219736099243, 0.5706844925880432, 0.7242632508277893], "r_norm": [0.24014504253864288, 0.2385360300540924, 0.24150380492210388], "ldr": {"dL_sdil": -0.0070352703332901, "dL_bp": -0.0109567791223526, "ratio": 0.6420929229957418}}, {"step": 4100, "epoch": 8, "train_loss": 0.02911919169127941, "cos_r_negg": [0.05804622545838356, 0.06820593774318695, 0.11597670614719391], "cos_innovation_negg": [0.5450683236122131, 0.5880045890808105, 0.7330162525177002], "cos_apical_negg": [0.05804622173309326, 0.06820593774318695, 0.1159767210483551], "cos_Ac_negg": [0.5449085831642151, 0.5885275602340698, 0.733569860458374], "r_norm": [0.2169014811515808, 0.21712006628513336, 0.22053925693035126]}, {"step": 4200, "epoch": 8, "train_loss": 0.0916910469532013, "cos_r_negg": [0.07937869429588318, 0.07797777652740479, 0.09943917393684387], "cos_innovation_negg": [0.5633472204208374, 0.6011552810668945, 0.7497129440307617], "cos_apical_negg": [0.07937869429588318, 0.07797776907682419, 0.09943918138742447], "cos_Ac_negg": [0.56303471326828, 0.6025720834732056, 0.7514515519142151], "r_norm": [0.2261510044336319, 0.2252989262342453, 0.22839811444282532]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9685, "test_loss": 0.12365431823730469}, {"step": 4300, "epoch": 9, "train_loss": 0.014758888632059097, "cos_r_negg": [0.0714099109172821, 0.06545557081699371, 0.1478579044342041], "cos_innovation_negg": [0.5366417169570923, 0.5749788284301758, 0.7624439001083374], "cos_apical_negg": [0.0714099109172821, 0.06545556336641312, 0.1478579193353653], "cos_Ac_negg": [0.5395289659500122, 0.5765002369880676, 0.761928141117096], "r_norm": [0.23815125226974487, 0.23755693435668945, 0.24205870926380157]}, {"step": 4400, "epoch": 9, "train_loss": 0.022416403517127037, "cos_r_negg": [0.06786151975393295, 0.08166593313217163, 0.16421091556549072], "cos_innovation_negg": [0.5520721673965454, 0.5914250016212463, 0.7781805992126465], "cos_apical_negg": [0.06786152720451355, 0.08166594058275223, 0.16421091556549072], "cos_Ac_negg": [0.5520327687263489, 0.5919884443283081, 0.778745174407959], "r_norm": [0.22056658565998077, 0.21906621754169464, 0.2203739881515503]}, {"step": 4500, "epoch": 9, "train_loss": 0.1037139743566513, "cos_r_negg": [0.0865529328584671, 0.09103463590145111, 0.14527535438537598], "cos_innovation_negg": [0.5638523101806641, 0.592156171798706, 0.7649362087249756], "cos_apical_negg": [0.0865529328584671, 0.09103463590145111, 0.14527535438537598], "cos_Ac_negg": [0.564079761505127, 0.5937734842300415, 0.7660852670669556], "r_norm": [0.22324925661087036, 0.22315490245819092, 0.22645336389541626], "ldr": {"dL_sdil": -0.008066743612289429, "dL_bp": -0.014248646795749664, "ratio": 0.5661410327537713}}, {"step": 4600, "epoch": 9, "train_loss": 0.06626339256763458, "cos_r_negg": [0.065360888838768, 0.060470327734947205, 0.12234711647033691], "cos_innovation_negg": [0.5448588132858276, 0.6093933582305908, 0.7503721714019775], "cos_apical_negg": [0.06536088138818741, 0.060470327734947205, 0.12234711647033691], "cos_Ac_negg": [0.5461468696594238, 0.610420823097229, 0.749968409538269], "r_norm": [0.22936847805976868, 0.22757463157176971, 0.22855454683303833]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9678, "test_loss": 0.12383741111755371}, {"step": 4700, "epoch": 10, "train_loss": 0.00938369520008564, "cos_r_negg": [0.037425555288791656, 0.06938180327415466, 0.11219051480293274], "cos_innovation_negg": [0.550470232963562, 0.5968233346939087, 0.7346566915512085], "cos_apical_negg": [0.03742556273937225, 0.06938180327415466, 0.11219051480293274], "cos_Ac_negg": [0.5507229566574097, 0.5979015827178955, 0.7344845533370972], "r_norm": [0.18589651584625244, 0.18497535586357117, 0.18600165843963623]}, {"step": 4800, "epoch": 10, "train_loss": 0.02552179805934429, "cos_r_negg": [0.06233280152082443, 0.07233235239982605, 0.1072329580783844], "cos_innovation_negg": [0.5499267578125, 0.5912210941314697, 0.7322781085968018], "cos_apical_negg": [0.06233280524611473, 0.07233234494924545, 0.1072329506278038], "cos_Ac_negg": [0.5506216287612915, 0.5924664735794067, 0.7314146757125854], "r_norm": [0.1988486349582672, 0.19887100160121918, 0.20252931118011475]}, {"step": 4900, "epoch": 10, "train_loss": 0.01581461913883686, "cos_r_negg": [0.05807516723871231, 0.06710566580295563, 0.12905558943748474], "cos_innovation_negg": [0.5676783323287964, 0.5948033332824707, 0.7567842602729797], "cos_apical_negg": [0.05807516723871231, 0.06710566580295563, 0.12905558943748474], "cos_Ac_negg": [0.5687493085861206, 0.5973098278045654, 0.7574092149734497], "r_norm": [0.2369706928730011, 0.23495718836784363, 0.23768897354602814]}, {"step": 5000, "epoch": 10, "train_loss": 0.014232782647013664, "cos_r_negg": [0.04947555810213089, 0.06722775101661682, 0.11993487179279327], "cos_innovation_negg": [0.5552632212638855, 0.6064639091491699, 0.7579385042190552], "cos_apical_negg": [0.04947555810213089, 0.06722775101661682, 0.11993487179279327], "cos_Ac_negg": [0.5565897226333618, 0.6067001819610596, 0.7580022811889648], "r_norm": [0.19451904296875, 0.19268527626991272, 0.1956268548965454], "ldr": {"dL_sdil": -0.007124781608581543, "dL_bp": -0.012223467230796814, "ratio": 0.5828773026552384}}, {"step": 5100, "epoch": 10, "train_loss": 0.03126565366983414, "cos_r_negg": [0.054448649287223816, 0.06965840607881546, 0.09593477845191956], "cos_innovation_negg": [0.5371192097663879, 0.5995650887489319, 0.7563377022743225], "cos_apical_negg": [0.054448649287223816, 0.06965841352939606, 0.09593477845191956], "cos_Ac_negg": [0.5426687002182007, 0.6089742183685303, 0.7673718929290771], "r_norm": [0.2243049293756485, 0.22551247477531433, 0.2285807728767395]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9682, "test_loss": 0.12618726348876952}, {"step": 5200, "epoch": 11, "train_loss": 0.013689623214304447, "cos_r_negg": [0.054590046405792236, 0.08569154888391495, 0.12170731276273727], "cos_innovation_negg": [0.5655243992805481, 0.6024428606033325, 0.7436208128929138], "cos_apical_negg": [0.054590046405792236, 0.08569155633449554, 0.12170730531215668], "cos_Ac_negg": [0.5670942068099976, 0.6042472124099731, 0.7451425790786743], "r_norm": [0.25497347116470337, 0.2538006901741028, 0.2551809847354889]}, {"step": 5300, "epoch": 11, "train_loss": 0.05269131064414978, "cos_r_negg": [0.05302569270133972, 0.06381505727767944, 0.13336099684238434], "cos_innovation_negg": [0.5120725631713867, 0.5659040808677673, 0.6957269906997681], "cos_apical_negg": [0.05302569270133972, 0.06381505727767944, 0.13336101174354553], "cos_Ac_negg": [0.51710045337677, 0.5723103880882263, 0.6966844797134399], "r_norm": [0.2222837507724762, 0.22246211767196655, 0.22576166689395905]}, {"step": 5400, "epoch": 11, "train_loss": 0.026194309815764427, "cos_r_negg": [0.052439652383327484, 0.06063498929142952, 0.10191938281059265], "cos_innovation_negg": [0.5543305277824402, 0.5927353501319885, 0.7473986148834229], "cos_apical_negg": [0.052439648658037186, 0.06063498929142952, 0.10191937536001205], "cos_Ac_negg": [0.5560851693153381, 0.5938558578491211, 0.7422469258308411], "r_norm": [0.19103989005088806, 0.19208963215351105, 0.19506944715976715]}, {"step": 5500, "epoch": 11, "train_loss": 0.03610093146562576, "cos_r_negg": [0.04882842302322388, 0.04626253619790077, 0.13743747770786285], "cos_innovation_negg": [0.525267481803894, 0.5838602781295776, 0.7465553283691406], "cos_apical_negg": [0.048828426748514175, 0.046262532472610474, 0.13743747770786285], "cos_Ac_negg": [0.5281760692596436, 0.5898793935775757, 0.7464447021484375], "r_norm": [0.19913125038146973, 0.19981563091278076, 0.2017751932144165], "ldr": {"dL_sdil": -0.0044263750314712524, "dL_bp": -0.00766851007938385, "ratio": 0.5772144765606024}}, {"step": 5600, "epoch": 11, "train_loss": 0.011690723709762096, "cos_r_negg": [0.04178202152252197, 0.04755251854658127, 0.13127601146697998], "cos_innovation_negg": [0.5479075312614441, 0.6049648523330688, 0.757026195526123], "cos_apical_negg": [0.041782017797231674, 0.04755251854658127, 0.13127601146697998], "cos_Ac_negg": [0.5566954016685486, 0.6144217848777771, 0.7610421180725098], "r_norm": [0.19590972363948822, 0.195388525724411, 0.19799646735191345]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9691, "test_loss": 0.13189056396484375}, {"step": 5700, "epoch": 12, "train_loss": 0.011434990912675858, "cos_r_negg": [0.04492928460240364, 0.04870538040995598, 0.11411378532648087], "cos_innovation_negg": [0.559892475605011, 0.614902138710022, 0.758762001991272], "cos_apical_negg": [0.04492928832769394, 0.04870537668466568, 0.11411378532648087], "cos_Ac_negg": [0.5645284652709961, 0.6230587959289551, 0.762065052986145], "r_norm": [0.19092899560928345, 0.18994152545928955, 0.19238710403442383]}, {"step": 5800, "epoch": 12, "train_loss": 0.01472583506256342, "cos_r_negg": [0.04657397419214249, 0.05546797066926956, 0.13012515008449554], "cos_innovation_negg": [0.5549585819244385, 0.6114252209663391, 0.7747702598571777], "cos_apical_negg": [0.04657397419214249, 0.05546797066926956, 0.13012516498565674], "cos_Ac_negg": [0.5605918765068054, 0.619053065776825, 0.7806684374809265], "r_norm": [0.21261551976203918, 0.21132779121398926, 0.2129901945590973]}, {"step": 5900, "epoch": 12, "train_loss": 0.027686260640621185, "cos_r_negg": [0.05407793074846268, 0.04876308888196945, 0.11269255727529526], "cos_innovation_negg": [0.5343650579452515, 0.5867140293121338, 0.7400655150413513], "cos_apical_negg": [0.05407793074846268, 0.04876308515667915, 0.11269255727529526], "cos_Ac_negg": [0.5467405319213867, 0.6034579277038574, 0.7492297887802124], "r_norm": [0.228031188249588, 0.22552496194839478, 0.22781668603420258]}, {"step": 6000, "epoch": 12, "train_loss": 0.006304215639829636, "cos_r_negg": [0.041400037705898285, 0.05253276973962784, 0.08973284810781479], "cos_innovation_negg": [0.5323338508605957, 0.5852477550506592, 0.7412900328636169], "cos_apical_negg": [0.04140004515647888, 0.05253276228904724, 0.08973284810781479], "cos_Ac_negg": [0.5374307632446289, 0.5913330316543579, 0.7445220351219177], "r_norm": [0.211392343044281, 0.20852413773536682, 0.20963749289512634], "ldr": {"dL_sdil": -0.008514508605003357, "dL_bp": -0.017385005950927734, "ratio": 0.489761615787597}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9696, "test_loss": 0.13102652072906495}, {"step": 6100, "epoch": 13, "train_loss": 0.007647733669728041, "cos_r_negg": [0.058543361723423004, 0.06763773411512375, 0.13031405210494995], "cos_innovation_negg": [0.5372574329376221, 0.5944938659667969, 0.7435393929481506], "cos_apical_negg": [0.058543361723423004, 0.06763774156570435, 0.13031405210494995], "cos_Ac_negg": [0.5454597473144531, 0.6029956340789795, 0.7495102882385254], "r_norm": [0.19791069626808167, 0.19725343585014343, 0.19918422400951385]}, {"step": 6200, "epoch": 13, "train_loss": 0.005339222028851509, "cos_r_negg": [0.03993219882249832, 0.0439935065805912, 0.12259697914123535], "cos_innovation_negg": [0.5299742817878723, 0.6072005033493042, 0.7573761940002441], "cos_apical_negg": [0.03993219509720802, 0.0439935028553009, 0.12259697169065475], "cos_Ac_negg": [0.5384821891784668, 0.6164162158966064, 0.7610653042793274], "r_norm": [0.2201082855463028, 0.21853888034820557, 0.22131897509098053]}, {"step": 6300, "epoch": 13, "train_loss": 0.020007651299238205, "cos_r_negg": [0.05164296180009842, 0.0499151386320591, 0.11171048134565353], "cos_innovation_negg": [0.5220210552215576, 0.578366219997406, 0.7537604570388794], "cos_apical_negg": [0.05164295434951782, 0.049915142357349396, 0.11171048879623413], "cos_Ac_negg": [0.5394889116287231, 0.5976953506469727, 0.764815092086792], "r_norm": [0.21189093589782715, 0.20991289615631104, 0.2125605046749115]}, {"step": 6400, "epoch": 13, "train_loss": 0.002438321942463517, "cos_r_negg": [0.05224587395787239, 0.05782228708267212, 0.13297517597675323], "cos_innovation_negg": [0.5443210601806641, 0.6096733212471008, 0.7620235681533813], "cos_apical_negg": [0.05224587023258209, 0.05782228708267212, 0.13297517597675323], "cos_Ac_negg": [0.5515585541725159, 0.6179340481758118, 0.7716501951217651], "r_norm": [0.19945526123046875, 0.19964176416397095, 0.2016235589981079]}, {"step": 6500, "epoch": 13, "train_loss": 0.002159678842872381, "cos_r_negg": [0.05180658400058746, 0.05799303948879242, 0.11578771471977234], "cos_innovation_negg": [0.5411338210105896, 0.5977074503898621, 0.7573133707046509], "cos_apical_negg": [0.05180658772587776, 0.05799303948879242, 0.11578771471977234], "cos_Ac_negg": [0.551083505153656, 0.6087433695793152, 0.765688419342041], "r_norm": [0.20487673580646515, 0.20408472418785095, 0.205448716878891], "ldr": {"dL_sdil": -0.007744446396827698, "dL_bp": -0.014109469950199127, "ratio": 0.5488828725786684}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9709, "test_loss": 0.12851683921813964}, {"step": 6600, "epoch": 14, "train_loss": 0.004894197452813387, "cos_r_negg": [0.047842200845479965, 0.04682867228984833, 0.10232914239168167], "cos_innovation_negg": [0.5222222805023193, 0.5879594087600708, 0.7287417650222778], "cos_apical_negg": [0.04784219712018967, 0.04682867228984833, 0.10232914239168167], "cos_Ac_negg": [0.5316355228424072, 0.599854588508606, 0.7331242561340332], "r_norm": [0.17443296313285828, 0.17286810278892517, 0.17643971741199493]}, {"step": 6700, "epoch": 14, "train_loss": 0.02553962916135788, "cos_r_negg": [0.0393855907022953, 0.05858352780342102, 0.1074909120798111], "cos_innovation_negg": [0.513884425163269, 0.570460319519043, 0.7442063689231873], "cos_apical_negg": [0.0393855907022953, 0.05858352780342102, 0.1074909120798111], "cos_Ac_negg": [0.5333433151245117, 0.5898236036300659, 0.7712637186050415], "r_norm": [0.19830980896949768, 0.19619552791118622, 0.19803929328918457]}, {"step": 6800, "epoch": 14, "train_loss": 0.004792890511453152, "cos_r_negg": [0.0455397292971611, 0.040460534393787384, 0.07618850469589233], "cos_innovation_negg": [0.535183310508728, 0.5900650024414062, 0.7600193619728088], "cos_apical_negg": [0.045539725571870804, 0.040460534393787384, 0.07618850469589233], "cos_Ac_negg": [0.5455622673034668, 0.603481650352478, 0.7685121297836304], "r_norm": [0.20605935156345367, 0.20373085141181946, 0.20592007040977478]}, {"step": 6900, "epoch": 14, "train_loss": 0.01101000141352415, "cos_r_negg": [0.051652900874614716, 0.0503389909863472, 0.08229105919599533], "cos_innovation_negg": [0.529188871383667, 0.5825726985931396, 0.75916987657547], "cos_apical_negg": [0.051652900874614716, 0.0503389909863472, 0.08229105919599533], "cos_Ac_negg": [0.5427979230880737, 0.5965732336044312, 0.7766227126121521], "r_norm": [0.21024341881275177, 0.20914150774478912, 0.21031688153743744]}, {"step": 7000, "epoch": 14, "train_loss": 0.03273762762546539, "cos_r_negg": [0.05334595963358879, 0.05724663287401199, 0.0838860496878624], "cos_innovation_negg": [0.5381057262420654, 0.5916114449501038, 0.7556002140045166], "cos_apical_negg": [0.05334595590829849, 0.057246625423431396, 0.083886057138443], "cos_Ac_negg": [0.5495507717132568, 0.6026450991630554, 0.7660986185073853], "r_norm": [0.21696007251739502, 0.2152629792690277, 0.21664190292358398], "ldr": {"dL_sdil": -0.006766654551029205, "dL_bp": -0.013199172914028168, "ratio": 0.5126574668809407}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.971, "test_loss": 0.13064061431884766}], "final": {"test_acc": 0.971, "test_loss": 0.13064061431884766, "wall_s": 25.36318016052246, "cos_r_negg": [0.059259649366140366, 0.056370340287685394, 0.07438581436872482], "cos_innovation_negg": [0.5242489576339722, 0.5776432752609253, 0.7376971244812012], "cos_apical_negg": [0.05925965681672096, 0.056370340287685394, 0.07438582181930542], "cos_Ac_negg": [0.5392985343933105, 0.5938428044319153, 0.7554187774658203]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_matched_s2.json b/results/nuis_ctrl_v1_rho0p05_matched_s2.json new file mode 100644 index 0000000..306ee2c --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_matched_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_matched_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.01640445366501808, 0.03720126673579216, 0.002073857933282852], "cos_innovation_negg": [0.016821179538965225, 0.03784848749637604, 0.002511262660846114], "cos_apical_negg": [0.01640445366501808, 0.03720126673579216, 0.002073857467621565], "cos_Ac_negg": [0.016821492463350296, 0.03784853219985962, 0.002511328551918268], "r_norm": [4.7503461837768555, 4.775265693664551, 4.857420921325684], "ldr": {"dL_sdil": -0.039087772369384766, "dL_bp": -0.24303245544433594, "ratio": 0.16083354915671916}}, {"step": 100, "epoch": 0, "train_loss": 0.3082430362701416, "cos_r_negg": [0.2857791781425476, 0.26766839623451233, 0.2713397741317749], "cos_innovation_negg": [0.5226379632949829, 0.4612053334712982, 0.404798686504364], "cos_apical_negg": [0.2857791781425476, 0.26766836643218994, 0.2713397741317749], "cos_Ac_negg": [0.5224592685699463, 0.46120312809944153, 0.404878169298172], "r_norm": [1.0421206951141357, 1.0571842193603516, 1.0707271099090576]}, {"step": 200, "epoch": 0, "train_loss": 0.36425015330314636, "cos_r_negg": [0.23564910888671875, 0.25292956829071045, 0.26488983631134033], "cos_innovation_negg": [0.5276224613189697, 0.5218232274055481, 0.5031354427337646], "cos_apical_negg": [0.23564910888671875, 0.25292956829071045, 0.26488983631134033], "cos_Ac_negg": [0.5273005962371826, 0.5217323303222656, 0.5031549334526062], "r_norm": [0.8230887651443481, 0.8372708559036255, 0.8474575877189636]}, {"step": 300, "epoch": 0, "train_loss": 0.25312429666519165, "cos_r_negg": [0.24125489592552185, 0.23980040848255157, 0.24009355902671814], "cos_innovation_negg": [0.5602648854255676, 0.5329376459121704, 0.5534262657165527], "cos_apical_negg": [0.24125489592552185, 0.23980042338371277, 0.24009355902671814], "cos_Ac_negg": [0.5597667694091797, 0.5328264236450195, 0.5531720519065857], "r_norm": [0.6523401737213135, 0.6598515510559082, 0.6712912321090698]}, {"step": 400, "epoch": 0, "train_loss": 0.1982860565185547, "cos_r_negg": [0.22769007086753845, 0.24220290780067444, 0.25251302123069763], "cos_innovation_negg": [0.5782732963562012, 0.550999104976654, 0.578500509262085], "cos_apical_negg": [0.22769007086753845, 0.24220290780067444, 0.25251302123069763], "cos_Ac_negg": [0.5780971050262451, 0.5510767698287964, 0.5779657363891602], "r_norm": [0.6497242450714111, 0.6555250287055969, 0.6676050424575806]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9408, "test_loss": 0.19557410736083986}, {"step": 500, "epoch": 1, "train_loss": 0.1452408730983734, "cos_r_negg": [0.19786790013313293, 0.21051079034805298, 0.26766398549079895], "cos_innovation_negg": [0.5602226257324219, 0.5618495941162109, 0.644468367099762], "cos_apical_negg": [0.19786790013313293, 0.21051079034805298, 0.26766401529312134], "cos_Ac_negg": [0.5643856525421143, 0.564034104347229, 0.6442623734474182], "r_norm": [0.5714954137802124, 0.5779463052749634, 0.5852220058441162], "ldr": {"dL_sdil": -0.017906367778778076, "dL_bp": -0.016708657145500183, "ratio": 1.071682040205155}}, {"step": 600, "epoch": 1, "train_loss": 0.09497902542352676, "cos_r_negg": [0.1796049177646637, 0.20196297764778137, 0.23781126737594604], "cos_innovation_negg": [0.5889536142349243, 0.5773766040802002, 0.680054783821106], "cos_apical_negg": [0.1796049028635025, 0.20196297764778137, 0.23781126737594604], "cos_Ac_negg": [0.5898095965385437, 0.5777332782745361, 0.6798214912414551], "r_norm": [0.4800269305706024, 0.48660504817962646, 0.4939706027507782]}, {"step": 700, "epoch": 1, "train_loss": 0.2234840989112854, "cos_r_negg": [0.18336820602416992, 0.18428264558315277, 0.26062068343162537], "cos_innovation_negg": [0.5697545409202576, 0.560819149017334, 0.670322060585022], "cos_apical_negg": [0.18336822092533112, 0.18428266048431396, 0.26062068343162537], "cos_Ac_negg": [0.570475161075592, 0.5621254444122314, 0.6705446243286133], "r_norm": [0.48902660608291626, 0.4951523542404175, 0.5056060552597046]}, {"step": 800, "epoch": 1, "train_loss": 0.11272813379764557, "cos_r_negg": [0.1483171433210373, 0.17584124207496643, 0.24780072271823883], "cos_innovation_negg": [0.5805045366287231, 0.5834454894065857, 0.6939709186553955], "cos_apical_negg": [0.1483171582221985, 0.17584124207496643, 0.24780073761940002], "cos_Ac_negg": [0.5813518166542053, 0.5840549468994141, 0.6939380168914795], "r_norm": [0.4260109066963196, 0.4302401542663574, 0.43916547298431396]}, {"step": 900, "epoch": 1, "train_loss": 0.2180328667163849, "cos_r_negg": [0.183700293302536, 0.2040768414735794, 0.2378573715686798], "cos_innovation_negg": [0.6245605945587158, 0.6176439523696899, 0.6703168153762817], "cos_apical_negg": [0.18370027840137482, 0.2040768563747406, 0.23785735666751862], "cos_Ac_negg": [0.6241723299026489, 0.6172389984130859, 0.6701363921165466], "r_norm": [0.4810108244419098, 0.48681724071502686, 0.4893745183944702]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9509, "test_loss": 0.16871641464233397}, {"step": 1000, "epoch": 2, "train_loss": 0.059693627059459686, "cos_r_negg": [0.16661351919174194, 0.19350527226924896, 0.21795561909675598], "cos_innovation_negg": [0.5953696370124817, 0.6116328835487366, 0.6691546440124512], "cos_apical_negg": [0.16661350429058075, 0.19350527226924896, 0.21795561909675598], "cos_Ac_negg": [0.5949282646179199, 0.6112121343612671, 0.6687310934066772], "r_norm": [0.46793025732040405, 0.4719570279121399, 0.48284944891929626], "ldr": {"dL_sdil": -0.01470687985420227, "dL_bp": -0.01627536118030548, "ratio": 0.903628478119355}}, {"step": 1100, "epoch": 2, "train_loss": 0.11942359805107117, "cos_r_negg": [0.1605493128299713, 0.20278623700141907, 0.20809486508369446], "cos_innovation_negg": [0.604766845703125, 0.6089346408843994, 0.6853097081184387], "cos_apical_negg": [0.1605493128299713, 0.20278623700141907, 0.20809486508369446], "cos_Ac_negg": [0.6044678092002869, 0.6086839437484741, 0.6849045753479004], "r_norm": [0.4059279263019562, 0.4114309549331665, 0.4171663224697113]}, {"step": 1200, "epoch": 2, "train_loss": 0.18524301052093506, "cos_r_negg": [0.15126459300518036, 0.1776367425918579, 0.21475918591022491], "cos_innovation_negg": [0.6136112213134766, 0.6074376702308655, 0.7134891152381897], "cos_apical_negg": [0.15126460790634155, 0.1776367425918579, 0.21475918591022491], "cos_Ac_negg": [0.6131654977798462, 0.6070705652236938, 0.7132871150970459], "r_norm": [0.422863245010376, 0.42861655354499817, 0.4363671541213989]}, {"step": 1300, "epoch": 2, "train_loss": 0.159089595079422, "cos_r_negg": [0.1309216320514679, 0.1585271954536438, 0.2243383228778839], "cos_innovation_negg": [0.5989238023757935, 0.5874024629592896, 0.6736501455307007], "cos_apical_negg": [0.1309216171503067, 0.1585271954536438, 0.2243383228778839], "cos_Ac_negg": [0.5985113382339478, 0.5871214270591736, 0.6733464002609253], "r_norm": [0.3669438660144806, 0.3706272840499878, 0.37795162200927734]}, {"step": 1400, "epoch": 2, "train_loss": 0.109891876578331, "cos_r_negg": [0.14230598509311676, 0.18260622024536133, 0.27371636033058167], "cos_innovation_negg": [0.5958492755889893, 0.5984202027320862, 0.6751723289489746], "cos_apical_negg": [0.14230598509311676, 0.18260622024536133, 0.27371636033058167], "cos_Ac_negg": [0.5951759815216064, 0.5978555083274841, 0.6748193502426147], "r_norm": [0.3571792542934418, 0.35940295457839966, 0.36750876903533936]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9582, "test_loss": 0.14771915092468263}, {"step": 1500, "epoch": 3, "train_loss": 0.07926538586616516, "cos_r_negg": [0.12977559864521027, 0.15848922729492188, 0.25051823258399963], "cos_innovation_negg": [0.6255969405174255, 0.6020369529724121, 0.660540759563446], "cos_apical_negg": [0.12977559864521027, 0.15848922729492188, 0.25051823258399963], "cos_Ac_negg": [0.6248027086257935, 0.6014535427093506, 0.6604979038238525], "r_norm": [0.33906564116477966, 0.34191954135894775, 0.34797239303588867], "ldr": {"dL_sdil": -0.008539244532585144, "dL_bp": -0.009055837988853455, "ratio": 0.942954649044719}}, {"step": 1600, "epoch": 3, "train_loss": 0.09568468481302261, "cos_r_negg": [0.1271316111087799, 0.14782167971134186, 0.23690281808376312], "cos_innovation_negg": [0.6143458485603333, 0.6216509342193604, 0.7200162410736084], "cos_apical_negg": [0.1271316111087799, 0.14782167971134186, 0.23690281808376312], "cos_Ac_negg": [0.6140349507331848, 0.6212351322174072, 0.7195559144020081], "r_norm": [0.3214662969112396, 0.32473963499069214, 0.33044254779815674]}, {"step": 1700, "epoch": 3, "train_loss": 0.06378612667322159, "cos_r_negg": [0.134951651096344, 0.15248778462409973, 0.20041069388389587], "cos_innovation_negg": [0.5914380550384521, 0.600161075592041, 0.685818612575531], "cos_apical_negg": [0.134951651096344, 0.15248778462409973, 0.20041069388389587], "cos_Ac_negg": [0.5911476612091064, 0.5998349189758301, 0.6854138374328613], "r_norm": [0.4012766480445862, 0.4040418863296509, 0.41464507579803467]}, {"step": 1800, "epoch": 3, "train_loss": 0.0223739892244339, "cos_r_negg": [0.116395965218544, 0.13231199979782104, 0.24531778693199158], "cos_innovation_negg": [0.5991317629814148, 0.6218805313110352, 0.7136843204498291], "cos_apical_negg": [0.116395965218544, 0.13231199979782104, 0.24531778693199158], "cos_Ac_negg": [0.598639726638794, 0.6211956143379211, 0.7116689682006836], "r_norm": [0.3940773904323578, 0.397135853767395, 0.4054872393608093]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9614, "test_loss": 0.1345835662841797}, {"step": 1900, "epoch": 4, "train_loss": 0.0561685636639595, "cos_r_negg": [0.10644853115081787, 0.11205698549747467, 0.22568316757678986], "cos_innovation_negg": [0.5882298946380615, 0.6098345518112183, 0.7257672548294067], "cos_apical_negg": [0.10644853115081787, 0.11205697804689407, 0.22568316757678986], "cos_Ac_negg": [0.5876476168632507, 0.609447717666626, 0.7250400185585022], "r_norm": [0.2725474238395691, 0.2741408944129944, 0.2807168662548065]}, {"step": 2000, "epoch": 4, "train_loss": 0.12074145674705505, "cos_r_negg": [0.10670299082994461, 0.16028940677642822, 0.2225094437599182], "cos_innovation_negg": [0.5697526931762695, 0.6004641652107239, 0.7349302768707275], "cos_apical_negg": [0.10670299828052521, 0.16028940677642822, 0.2225094437599182], "cos_Ac_negg": [0.5688590407371521, 0.5997929573059082, 0.7337663769721985], "r_norm": [0.33775901794433594, 0.3393515646457672, 0.35109880566596985], "ldr": {"dL_sdil": -0.009059593081474304, "dL_bp": -0.011021822690963745, "ratio": 0.8219686829930428}}, {"step": 2100, "epoch": 4, "train_loss": 0.05476764589548111, "cos_r_negg": [0.11422640830278397, 0.1464477777481079, 0.21328723430633545], "cos_innovation_negg": [0.6073275804519653, 0.6298151016235352, 0.7264124155044556], "cos_apical_negg": [0.11422640830278397, 0.1464477777481079, 0.21328723430633545], "cos_Ac_negg": [0.6067020893096924, 0.6292158365249634, 0.7263119220733643], "r_norm": [0.32948994636535645, 0.3308507204055786, 0.33968520164489746]}, {"step": 2200, "epoch": 4, "train_loss": 0.03519546240568161, "cos_r_negg": [0.10570386052131653, 0.1540318876504898, 0.19078221917152405], "cos_innovation_negg": [0.6084020137786865, 0.635105311870575, 0.7261720895767212], "cos_apical_negg": [0.10570386052131653, 0.1540318876504898, 0.19078223407268524], "cos_Ac_negg": [0.6071482300758362, 0.6345615983009338, 0.7255818247795105], "r_norm": [0.30364125967025757, 0.30542615056037903, 0.3137291669845581]}, {"step": 2300, "epoch": 4, "train_loss": 0.15517012774944305, "cos_r_negg": [0.08970008790493011, 0.12925715744495392, 0.1617332249879837], "cos_innovation_negg": [0.5841630697250366, 0.6309168934822083, 0.7412790656089783], "cos_apical_negg": [0.08970008790493011, 0.12925715744495392, 0.1617332398891449], "cos_Ac_negg": [0.5836247205734253, 0.6307811737060547, 0.7405720949172974], "r_norm": [0.29441773891448975, 0.29791921377182007, 0.30677637457847595]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9641, "test_loss": 0.12694887351989745}, {"step": 2400, "epoch": 5, "train_loss": 0.10991528630256653, "cos_r_negg": [0.08241323381662369, 0.1163623034954071, 0.1318112164735794], "cos_innovation_negg": [0.573365330696106, 0.6151907444000244, 0.7094967365264893], "cos_apical_negg": [0.08241324126720428, 0.1163623109459877, 0.1318112164735794], "cos_Ac_negg": [0.572695255279541, 0.6145823001861572, 0.7089481353759766], "r_norm": [0.28294146060943604, 0.2839856445789337, 0.2929994761943817]}, {"step": 2500, "epoch": 5, "train_loss": 0.028982385993003845, "cos_r_negg": [0.08597160130739212, 0.11241570115089417, 0.1342657506465912], "cos_innovation_negg": [0.5748742818832397, 0.6111358404159546, 0.7245398759841919], "cos_apical_negg": [0.08597160130739212, 0.11241571605205536, 0.13426576554775238], "cos_Ac_negg": [0.5743154287338257, 0.6104971170425415, 0.7234162092208862], "r_norm": [0.2821938097476959, 0.2846311330795288, 0.2924949824810028], "ldr": {"dL_sdil": -0.00847238302230835, "dL_bp": -0.011189520359039307, "ratio": 0.7571712415236858}}, {"step": 2600, "epoch": 5, "train_loss": 0.07748591899871826, "cos_r_negg": [0.1086062639951706, 0.1370469182729721, 0.21421417593955994], "cos_innovation_negg": [0.6121000051498413, 0.6413086652755737, 0.7335861921310425], "cos_apical_negg": [0.10860625654459, 0.1370469182729721, 0.21421417593955994], "cos_Ac_negg": [0.6116146445274353, 0.6407825946807861, 0.7327148914337158], "r_norm": [0.28830447793006897, 0.2895393967628479, 0.29767265915870667]}, {"step": 2700, "epoch": 5, "train_loss": 0.0351409837603569, "cos_r_negg": [0.0786549299955368, 0.13360951840877533, 0.16454938054084778], "cos_innovation_negg": [0.6102995872497559, 0.6249126195907593, 0.7394830584526062], "cos_apical_negg": [0.0786549299955368, 0.13360951840877533, 0.16454938054084778], "cos_Ac_negg": [0.6099395751953125, 0.6243188977241516, 0.7385637760162354], "r_norm": [0.25944003462791443, 0.26124751567840576, 0.2700679302215576]}, {"step": 2800, "epoch": 5, "train_loss": 0.15840259194374084, "cos_r_negg": [0.09352105855941772, 0.1378842294216156, 0.22651061415672302], "cos_innovation_negg": [0.601628839969635, 0.62410968542099, 0.7716900110244751], "cos_apical_negg": [0.09352105855941772, 0.1378841996192932, 0.22651061415672302], "cos_Ac_negg": [0.6006388664245605, 0.6229953765869141, 0.7707330584526062], "r_norm": [0.2898827791213989, 0.2916167676448822, 0.3004188537597656]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9625, "test_loss": 0.13689499206542968}, {"step": 2900, "epoch": 6, "train_loss": 0.010221228003501892, "cos_r_negg": [0.06756944209337234, 0.1255694329738617, 0.14716842770576477], "cos_innovation_negg": [0.5909930467605591, 0.6306226253509521, 0.7495819926261902], "cos_apical_negg": [0.06756944209337234, 0.1255694329738617, 0.14716842770576477], "cos_Ac_negg": [0.5920403003692627, 0.6320813894271851, 0.7511036396026611], "r_norm": [0.3089599311351776, 0.31268855929374695, 0.3226047158241272]}, {"step": 3000, "epoch": 6, "train_loss": 0.19828207790851593, "cos_r_negg": [0.05993488058447838, 0.10396704077720642, 0.12716127932071686], "cos_innovation_negg": [0.5931680798530579, 0.6245844960212708, 0.7405874729156494], "cos_apical_negg": [0.05993488430976868, 0.10396703332662582, 0.12716126441955566], "cos_Ac_negg": [0.5925907492637634, 0.6237685680389404, 0.7394886016845703], "r_norm": [0.2809411883354187, 0.2814246416091919, 0.2905486822128296], "ldr": {"dL_sdil": -0.010096311569213867, "dL_bp": -0.01543637365102768, "ratio": 0.6540598068861662}}, {"step": 3100, "epoch": 6, "train_loss": 0.06638136506080627, "cos_r_negg": [0.07229498028755188, 0.11040383577346802, 0.15499883890151978], "cos_innovation_negg": [0.6182936429977417, 0.6437310576438904, 0.7536247372627258], "cos_apical_negg": [0.07229498028755188, 0.11040383577346802, 0.15499883890151978], "cos_Ac_negg": [0.617737889289856, 0.6430108547210693, 0.7527674436569214], "r_norm": [0.2609640061855316, 0.26230838894844055, 0.26992547512054443]}, {"step": 3200, "epoch": 6, "train_loss": 0.0991695374250412, "cos_r_negg": [0.061629168689250946, 0.12327949702739716, 0.14487981796264648], "cos_innovation_negg": [0.6122725009918213, 0.6322770714759827, 0.7382808923721313], "cos_apical_negg": [0.061629172414541245, 0.12327949702739716, 0.14487981796264648], "cos_Ac_negg": [0.6113296151161194, 0.6308178901672363, 0.7371530532836914], "r_norm": [0.2579784393310547, 0.25766634941101074, 0.26582401990890503]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9662, "test_loss": 0.1270959690093994}, {"step": 3300, "epoch": 7, "train_loss": 0.06997574120759964, "cos_r_negg": [0.07369835674762726, 0.08560314774513245, 0.1495373398065567], "cos_innovation_negg": [0.5973509550094604, 0.6290678381919861, 0.7434496283531189], "cos_apical_negg": [0.07369835674762726, 0.08560314774513245, 0.1495373398065567], "cos_Ac_negg": [0.5967786312103271, 0.6273025274276733, 0.7424504160881042], "r_norm": [0.23922085762023926, 0.23994746804237366, 0.24675823748111725]}, {"step": 3400, "epoch": 7, "train_loss": 0.0835975781083107, "cos_r_negg": [0.057529304176568985, 0.09213370829820633, 0.10100296139717102], "cos_innovation_negg": [0.604116678237915, 0.6394843459129333, 0.747037410736084], "cos_apical_negg": [0.05752930790185928, 0.09213370829820633, 0.10100294649600983], "cos_Ac_negg": [0.6032308340072632, 0.6386922597885132, 0.7457476854324341], "r_norm": [0.2492670714855194, 0.2504112422466278, 0.2582710087299347]}, {"step": 3500, "epoch": 7, "train_loss": 0.04674973338842392, "cos_r_negg": [0.06157372146844864, 0.10763275623321533, 0.08638191223144531], "cos_innovation_negg": [0.5806431770324707, 0.6161267757415771, 0.7198772430419922], "cos_apical_negg": [0.06157372146844864, 0.10763275623321533, 0.08638191223144531], "cos_Ac_negg": [0.5795246958732605, 0.6152815818786621, 0.7186082601547241], "r_norm": [0.251146137714386, 0.2520167827606201, 0.25975173711776733], "ldr": {"dL_sdil": -0.006019622087478638, "dL_bp": -0.010309994220733643, "ratio": 0.5838627993964376}}, {"step": 3600, "epoch": 7, "train_loss": 0.03467125445604324, "cos_r_negg": [0.05732931196689606, 0.08136971294879913, 0.11719461530447006], "cos_innovation_negg": [0.5794965028762817, 0.6180660724639893, 0.7294912338256836], "cos_apical_negg": [0.05732931196689606, 0.08136971294879913, 0.11719462275505066], "cos_Ac_negg": [0.5781562328338623, 0.6155945062637329, 0.7301044464111328], "r_norm": [0.2593074440956116, 0.2598908245563507, 0.2697942554950714]}, {"step": 3700, "epoch": 7, "train_loss": 0.052790433168411255, "cos_r_negg": [0.07237672060728073, 0.09659852832555771, 0.16876760125160217], "cos_innovation_negg": [0.6170052289962769, 0.6482290029525757, 0.7726831436157227], "cos_apical_negg": [0.07237672060728073, 0.0965985357761383, 0.16876761615276337], "cos_Ac_negg": [0.6160683631896973, 0.6473923325538635, 0.7716935873031616], "r_norm": [0.24569517374038696, 0.24590174853801727, 0.2529482841491699]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9621, "test_loss": 0.13811623802185058}, {"step": 3800, "epoch": 8, "train_loss": 0.027530092746019363, "cos_r_negg": [0.05002554506063461, 0.09452114999294281, 0.13623180985450745], "cos_innovation_negg": [0.6077260971069336, 0.6403964757919312, 0.7709159851074219], "cos_apical_negg": [0.05002554506063461, 0.09452114999294281, 0.13623180985450745], "cos_Ac_negg": [0.6076191663742065, 0.6401568651199341, 0.7717118859291077], "r_norm": [0.22229140996932983, 0.2229335606098175, 0.2293362319469452]}, {"step": 3900, "epoch": 8, "train_loss": 0.07226866483688354, "cos_r_negg": [0.04363337159156799, 0.08218210935592651, 0.1265658438205719], "cos_innovation_negg": [0.615979790687561, 0.6455563902854919, 0.7541069984436035], "cos_apical_negg": [0.04363337159156799, 0.08218210935592651, 0.1265658438205719], "cos_Ac_negg": [0.6151710748672485, 0.6447477340698242, 0.7533867359161377], "r_norm": [0.21910634636878967, 0.21962423622608185, 0.22751544415950775]}, {"step": 4000, "epoch": 8, "train_loss": 0.0318903848528862, "cos_r_negg": [0.05057606101036072, 0.09977037459611893, 0.08854591101408005], "cos_innovation_negg": [0.6203804016113281, 0.6429596543312073, 0.7605069875717163], "cos_apical_negg": [0.05057606101036072, 0.09977037459611893, 0.08854591101408005], "cos_Ac_negg": [0.6203014254570007, 0.6421242952346802, 0.7587669491767883], "r_norm": [0.22447727620601654, 0.22519220411777496, 0.23202501237392426], "ldr": {"dL_sdil": -0.00527597963809967, "dL_bp": -0.008125081658363342, "ratio": 0.649344813989528}}, {"step": 4100, "epoch": 8, "train_loss": 0.03050089254975319, "cos_r_negg": [0.047305360436439514, 0.10678502172231674, 0.1110907718539238], "cos_innovation_negg": [0.6034614443778992, 0.6402506232261658, 0.7637248039245605], "cos_apical_negg": [0.04730536416172981, 0.10678501427173615, 0.1110907718539238], "cos_Ac_negg": [0.6041349768638611, 0.6407039165496826, 0.7633804082870483], "r_norm": [0.2699483036994934, 0.2691279351711273, 0.2764284014701843]}, {"step": 4200, "epoch": 8, "train_loss": 0.043460726737976074, "cos_r_negg": [0.04512732848525047, 0.08147674798965454, 0.08984723687171936], "cos_innovation_negg": [0.6105595231056213, 0.6448302268981934, 0.772793710231781], "cos_apical_negg": [0.04512732848525047, 0.08147674798965454, 0.08984723687171936], "cos_Ac_negg": [0.6104952692985535, 0.6454557180404663, 0.7713298797607422], "r_norm": [0.2247942090034485, 0.22512578964233398, 0.23246988654136658]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9693, "test_loss": 0.12154156303405762}, {"step": 4300, "epoch": 9, "train_loss": 0.030666926875710487, "cos_r_negg": [0.043922558426856995, 0.08970377594232559, 0.12721070647239685], "cos_innovation_negg": [0.5748286843299866, 0.6414573788642883, 0.7450218200683594], "cos_apical_negg": [0.043922558426856995, 0.089703768491745, 0.12721069157123566], "cos_Ac_negg": [0.5760213136672974, 0.6418507099151611, 0.7453689575195312], "r_norm": [0.23060771822929382, 0.22993126511573792, 0.23822590708732605]}, {"step": 4400, "epoch": 9, "train_loss": 0.009681368246674538, "cos_r_negg": [0.02731999009847641, 0.08435576409101486, 0.10899575054645538], "cos_innovation_negg": [0.5828015804290771, 0.6213865280151367, 0.733077883720398], "cos_apical_negg": [0.02731999196112156, 0.08435575664043427, 0.10899574309587479], "cos_Ac_negg": [0.5844757556915283, 0.6216385960578918, 0.7338995933532715], "r_norm": [0.210330069065094, 0.21000352501869202, 0.21758881211280823]}, {"step": 4500, "epoch": 9, "train_loss": 0.16826480627059937, "cos_r_negg": [0.03209655359387398, 0.09068798273801804, 0.12058517336845398], "cos_innovation_negg": [0.6033183336257935, 0.6339293718338013, 0.7712758779525757], "cos_apical_negg": [0.03209655359387398, 0.09068797528743744, 0.12058517336845398], "cos_Ac_negg": [0.6031326651573181, 0.6329625248908997, 0.7699393033981323], "r_norm": [0.22477683424949646, 0.22498223185539246, 0.23199597001075745], "ldr": {"dL_sdil": -0.005679681897163391, "dL_bp": -0.009906232357025146, "ratio": 0.5733443041173533}}, {"step": 4600, "epoch": 9, "train_loss": 0.022005319595336914, "cos_r_negg": [0.049900319427251816, 0.09338614344596863, 0.10384596884250641], "cos_innovation_negg": [0.6106094121932983, 0.6418677568435669, 0.7661275863647461], "cos_apical_negg": [0.04990031570196152, 0.09338614344596863, 0.10384596884250641], "cos_Ac_negg": [0.611644446849823, 0.6402194499969482, 0.7652423977851868], "r_norm": [0.24207225441932678, 0.24283793568611145, 0.2503352165222168]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9676, "test_loss": 0.12289777679443359}, {"step": 4700, "epoch": 10, "train_loss": 0.01217680238187313, "cos_r_negg": [0.044108569622039795, 0.07891270518302917, 0.10455731302499771], "cos_innovation_negg": [0.5981069803237915, 0.6320803761482239, 0.7616831064224243], "cos_apical_negg": [0.044108569622039795, 0.07891270518302917, 0.10455730557441711], "cos_Ac_negg": [0.5982277989387512, 0.6322939395904541, 0.7608315348625183], "r_norm": [0.23162482678890228, 0.23281437158584595, 0.2404969185590744]}, {"step": 4800, "epoch": 10, "train_loss": 0.039784226566553116, "cos_r_negg": [0.058429427444934845, 0.05808219313621521, 0.07744204998016357], "cos_innovation_negg": [0.6007778644561768, 0.6256026029586792, 0.7507044076919556], "cos_apical_negg": [0.058429427444934845, 0.05808219313621521, 0.07744204998016357], "cos_Ac_negg": [0.6001363396644592, 0.6244678497314453, 0.750571072101593], "r_norm": [0.20747005939483643, 0.20756027102470398, 0.21459421515464783]}, {"step": 4900, "epoch": 10, "train_loss": 0.021471809595823288, "cos_r_negg": [0.04894174635410309, 0.05147011578083038, 0.08570066094398499], "cos_innovation_negg": [0.604030966758728, 0.6277135610580444, 0.7654829025268555], "cos_apical_negg": [0.04894174262881279, 0.05147011578083038, 0.08570065349340439], "cos_Ac_negg": [0.6059499979019165, 0.6286013722419739, 0.7663230895996094], "r_norm": [0.21760371327400208, 0.21674123406410217, 0.22461354732513428]}, {"step": 5000, "epoch": 10, "train_loss": 0.02022964134812355, "cos_r_negg": [0.04769568145275116, 0.07722481340169907, 0.08362030237913132], "cos_innovation_negg": [0.5931369066238403, 0.6184937953948975, 0.7711560130119324], "cos_apical_negg": [0.04769568517804146, 0.07722481340169907, 0.08362030237913132], "cos_Ac_negg": [0.5934152007102966, 0.6178797483444214, 0.7690629959106445], "r_norm": [0.22270342707633972, 0.2225886434316635, 0.22867533564567566], "ldr": {"dL_sdil": -0.004315398633480072, "dL_bp": -0.006797440350055695, "ratio": 0.6348564181875777}}, {"step": 5100, "epoch": 10, "train_loss": 0.06920983642339706, "cos_r_negg": [0.04221167415380478, 0.07960735261440277, 0.07969915121793747], "cos_innovation_negg": [0.5961176753044128, 0.6356909871101379, 0.7837827801704407], "cos_apical_negg": [0.04221167415380478, 0.07960735261440277, 0.07969914376735687], "cos_Ac_negg": [0.596979558467865, 0.6343303322792053, 0.7813828587532043], "r_norm": [0.19715000689029694, 0.1968795657157898, 0.20344626903533936]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9658, "test_loss": 0.1333168104171753}, {"step": 5200, "epoch": 11, "train_loss": 0.012186631560325623, "cos_r_negg": [0.0501217395067215, 0.07426240295171738, 0.08266233652830124], "cos_innovation_negg": [0.6005086898803711, 0.6364537477493286, 0.7799835205078125], "cos_apical_negg": [0.0501217395067215, 0.07426239550113678, 0.08266233652830124], "cos_Ac_negg": [0.6019777655601501, 0.6354715824127197, 0.778002142906189], "r_norm": [0.21032944321632385, 0.2092743068933487, 0.2160453498363495]}, {"step": 5300, "epoch": 11, "train_loss": 0.07353296875953674, "cos_r_negg": [0.04123096540570259, 0.08277693390846252, 0.04575967416167259], "cos_innovation_negg": [0.6052377223968506, 0.6304951310157776, 0.7596474885940552], "cos_apical_negg": [0.04123096540570259, 0.08277693390846252, 0.04575967416167259], "cos_Ac_negg": [0.6058896780014038, 0.6270653605461121, 0.7583368420600891], "r_norm": [0.21540212631225586, 0.21425561606884003, 0.22190037369728088]}, {"step": 5400, "epoch": 11, "train_loss": 0.024054862558841705, "cos_r_negg": [0.03200096637010574, 0.05857172980904579, 0.06204779073596001], "cos_innovation_negg": [0.5978760719299316, 0.6196624040603638, 0.7694088816642761], "cos_apical_negg": [0.03200096637010574, 0.05857173353433609, 0.06204779073596001], "cos_Ac_negg": [0.5994182825088501, 0.6172158122062683, 0.7662521004676819], "r_norm": [0.19756881892681122, 0.19676034152507782, 0.20345748960971832]}, {"step": 5500, "epoch": 11, "train_loss": 0.02114860899746418, "cos_r_negg": [0.03341823071241379, 0.07660919427871704, 0.05777721479535103], "cos_innovation_negg": [0.5927249193191528, 0.6290849447250366, 0.7800296545028687], "cos_apical_negg": [0.03341822698712349, 0.07660919427871704, 0.05777721479535103], "cos_Ac_negg": [0.5956510305404663, 0.6277630925178528, 0.7773447632789612], "r_norm": [0.1989668607711792, 0.19816920161247253, 0.2042587250471115], "ldr": {"dL_sdil": -0.00471380352973938, "dL_bp": -0.009600669145584106, "ratio": 0.4909869779136724}}, {"step": 5600, "epoch": 11, "train_loss": 0.019959719851613045, "cos_r_negg": [0.03105173259973526, 0.06734295189380646, 0.04771738499403], "cos_innovation_negg": [0.5900029540061951, 0.6324846744537354, 0.7677162885665894], "cos_apical_negg": [0.03105173259973526, 0.06734294444322586, 0.0477173775434494], "cos_Ac_negg": [0.5941534638404846, 0.6328752040863037, 0.7669576406478882], "r_norm": [0.2001800388097763, 0.1997605562210083, 0.20639798045158386]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9673, "test_loss": 0.13266973419189454}, {"step": 5700, "epoch": 12, "train_loss": 0.060172341763973236, "cos_r_negg": [0.019513752311468124, 0.06887805461883545, 0.052588433027267456], "cos_innovation_negg": [0.6081061363220215, 0.6361182928085327, 0.7526764869689941], "cos_apical_negg": [0.019513752311468124, 0.06887805461883545, 0.052588436752557755], "cos_Ac_negg": [0.6076500415802002, 0.6346127390861511, 0.7518815398216248], "r_norm": [0.18911883234977722, 0.1886226236820221, 0.1952684372663498]}, {"step": 5800, "epoch": 12, "train_loss": 0.014476735144853592, "cos_r_negg": [0.016671661287546158, 0.06142142787575722, 0.042581893503665924], "cos_innovation_negg": [0.604634165763855, 0.6325477361679077, 0.7547914981842041], "cos_apical_negg": [0.016671663150191307, 0.06142142787575722, 0.042581893503665924], "cos_Ac_negg": [0.6081270575523376, 0.6317968964576721, 0.7564231753349304], "r_norm": [0.2067956030368805, 0.20641526579856873, 0.21309533715248108]}, {"step": 5900, "epoch": 12, "train_loss": 0.01700747013092041, "cos_r_negg": [0.02572738379240036, 0.05525951087474823, 0.0465284064412117], "cos_innovation_negg": [0.6017051935195923, 0.6358292102813721, 0.766609787940979], "cos_apical_negg": [0.02572738379240036, 0.05525950342416763, 0.0465284064412117], "cos_Ac_negg": [0.6069463491439819, 0.635897696018219, 0.7707664966583252], "r_norm": [0.2125571370124817, 0.211478590965271, 0.2183077037334442]}, {"step": 6000, "epoch": 12, "train_loss": 0.01322426088154316, "cos_r_negg": [0.03148920461535454, 0.05611826479434967, 0.03532717376947403], "cos_innovation_negg": [0.5929558277130127, 0.6213021278381348, 0.7593656778335571], "cos_apical_negg": [0.031489208340644836, 0.05611826479434967, 0.035327181220054626], "cos_Ac_negg": [0.5988937616348267, 0.621841311454773, 0.7613189816474915], "r_norm": [0.2208387553691864, 0.22004424035549164, 0.22557605803012848], "ldr": {"dL_sdil": -0.006456539034843445, "dL_bp": -0.011764660477638245, "ratio": 0.5488079360314523}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9675, "test_loss": 0.13005401153564453}, {"step": 6100, "epoch": 13, "train_loss": 0.022220883518457413, "cos_r_negg": [0.03963175043463707, 0.052392616868019104, 0.030986912548542023], "cos_innovation_negg": [0.6018118262290955, 0.6285805702209473, 0.7574760317802429], "cos_apical_negg": [0.039631739258766174, 0.052392616868019104, 0.030986912548542023], "cos_Ac_negg": [0.606399655342102, 0.6306541562080383, 0.7582318782806396], "r_norm": [0.1958753764629364, 0.1944025456905365, 0.2011382281780243]}, {"step": 6200, "epoch": 13, "train_loss": 0.0215994194149971, "cos_r_negg": [0.022252589464187622, 0.058879636228084564, 0.05355587974190712], "cos_innovation_negg": [0.5997023582458496, 0.6299881935119629, 0.7478371858596802], "cos_apical_negg": [0.02225259318947792, 0.058879632502794266, 0.05355587974190712], "cos_Ac_negg": [0.6039952039718628, 0.6308549046516418, 0.7515855431556702], "r_norm": [0.19756633043289185, 0.19615325331687927, 0.20227526128292084]}, {"step": 6300, "epoch": 13, "train_loss": 0.008365434594452381, "cos_r_negg": [0.03482650965452194, 0.05872643366456032, 0.04233097285032272], "cos_innovation_negg": [0.6048058271408081, 0.6315643787384033, 0.7636400461196899], "cos_apical_negg": [0.034826505929231644, 0.05872643366456032, 0.04233097657561302], "cos_Ac_negg": [0.6079874038696289, 0.6312144994735718, 0.7649940252304077], "r_norm": [0.19982211291790009, 0.19874870777130127, 0.20558907091617584]}, {"step": 6400, "epoch": 13, "train_loss": 0.017093073576688766, "cos_r_negg": [0.024669818580150604, 0.06180482357740402, 0.03846380487084389], "cos_innovation_negg": [0.6127402782440186, 0.6329571604728699, 0.7652629613876343], "cos_apical_negg": [0.024669822305440903, 0.06180482730269432, 0.038463808596134186], "cos_Ac_negg": [0.616874098777771, 0.632465660572052, 0.7677662968635559], "r_norm": [0.18896402418613434, 0.18796700239181519, 0.1949893981218338]}, {"step": 6500, "epoch": 13, "train_loss": 0.002227767137810588, "cos_r_negg": [0.02520844154059887, 0.050781913101673126, 0.020121363922953606], "cos_innovation_negg": [0.5908157825469971, 0.6231894493103027, 0.7754924297332764], "cos_apical_negg": [0.02520844340324402, 0.050781916826963425, 0.020121369510889053], "cos_Ac_negg": [0.5939737558364868, 0.6233177185058594, 0.7799904346466064], "r_norm": [0.1786518096923828, 0.17739295959472656, 0.18380557000637054], "ldr": {"dL_sdil": -0.004798531532287598, "dL_bp": -0.009442314505577087, "ratio": 0.5081944187998984}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9689, "test_loss": 0.1358599048614502}, {"step": 6600, "epoch": 14, "train_loss": 0.005903652869164944, "cos_r_negg": [0.021855976432561874, 0.05522775650024414, 0.04849259555339813], "cos_innovation_negg": [0.5924265384674072, 0.6146969795227051, 0.7547267079353333], "cos_apical_negg": [0.021855976432561874, 0.05522775650024414, 0.04849259555339813], "cos_Ac_negg": [0.5953912734985352, 0.6167807579040527, 0.7582675218582153], "r_norm": [0.18992452323436737, 0.1884787231683731, 0.1949426531791687]}, {"step": 6700, "epoch": 14, "train_loss": 0.005318871699273586, "cos_r_negg": [0.01903548091650009, 0.06140562891960144, 0.01907830685377121], "cos_innovation_negg": [0.584149181842804, 0.6265344023704529, 0.7656687498092651], "cos_apical_negg": [0.01903548091650009, 0.06140563637018204, 0.01907830312848091], "cos_Ac_negg": [0.588869571685791, 0.6270227432250977, 0.7711664438247681], "r_norm": [0.20989686250686646, 0.20838533341884613, 0.2147705852985382]}, {"step": 6800, "epoch": 14, "train_loss": 0.014734535478055477, "cos_r_negg": [0.021413415670394897, 0.06165187060832977, 0.050687797367572784], "cos_innovation_negg": [0.5898095965385437, 0.6341552734375, 0.7743579745292664], "cos_apical_negg": [0.021413417533040047, 0.06165187060832977, 0.050687797367572784], "cos_Ac_negg": [0.5935208797454834, 0.6355680227279663, 0.775898814201355], "r_norm": [0.198848158121109, 0.19746097922325134, 0.20446176826953888]}, {"step": 6900, "epoch": 14, "train_loss": 0.006750402972102165, "cos_r_negg": [0.021260786801576614, 0.06827224045991898, 0.03903006389737129], "cos_innovation_negg": [0.5947901010513306, 0.6297394037246704, 0.7735829949378967], "cos_apical_negg": [0.021260792389512062, 0.06827224045991898, 0.03903006762266159], "cos_Ac_negg": [0.6018731594085693, 0.6337403655052185, 0.7811018228530884], "r_norm": [0.2032964527606964, 0.20185834169387817, 0.20878508687019348]}, {"step": 7000, "epoch": 14, "train_loss": 0.009256819263100624, "cos_r_negg": [0.03031407669186592, 0.06538739800453186, 0.03387116640806198], "cos_innovation_negg": [0.5915297269821167, 0.6181275844573975, 0.7697106599807739], "cos_apical_negg": [0.030314071103930473, 0.06538739800453186, 0.03387116268277168], "cos_Ac_negg": [0.5992513298988342, 0.6193030476570129, 0.7756493091583252], "r_norm": [0.19728296995162964, 0.19664275646209717, 0.20322078466415405], "ldr": {"dL_sdil": -0.0067901611328125, "dL_bp": -0.013934463262557983, "ratio": 0.4872926215290774}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9677, "test_loss": 0.14392237319946288}], "final": {"test_acc": 0.9677, "test_loss": 0.14392237319946288, "wall_s": 25.392999172210693, "cos_r_negg": [0.0266385767608881, 0.05587003007531166, 0.038269370794296265], "cos_innovation_negg": [0.5914689898490906, 0.6164125800132751, 0.7689792513847351], "cos_apical_negg": [0.026638582348823547, 0.05587003380060196, 0.038269370794296265], "cos_Ac_negg": [0.5985291600227356, 0.6189343929290771, 0.772524893283844]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_matched_s3.json b/results/nuis_ctrl_v1_rho0p05_matched_s3.json new file mode 100644 index 0000000..d2533dd --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_matched_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_matched_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.015490960329771042, 0.01314460951834917, 0.005123969167470932], "cos_innovation_negg": [-0.014851665124297142, 0.013897672295570374, 0.00600943760946393], "cos_apical_negg": [-0.015490962192416191, 0.013144610449671745, 0.005123969167470932], "cos_Ac_negg": [-0.014851845800876617, 0.013897630386054516, 0.006009321194142103], "r_norm": [4.857795238494873, 4.751279354095459, 4.85173225402832], "ldr": {"dL_sdil": -0.0007169246673583984, "dL_bp": -0.22743678092956543, "ratio": 0.003152193169584219}}, {"step": 100, "epoch": 0, "train_loss": 0.33210813999176025, "cos_r_negg": [0.2446061372756958, 0.24338620901107788, 0.22636139392852783], "cos_innovation_negg": [0.48129013180732727, 0.44585877656936646, 0.3610212504863739], "cos_apical_negg": [0.2446061372756958, 0.24338620901107788, 0.22636139392852783], "cos_Ac_negg": [0.4819962978363037, 0.44557255506515503, 0.36056792736053467], "r_norm": [0.9718714952468872, 0.9574735164642334, 0.9659876227378845]}, {"step": 200, "epoch": 0, "train_loss": 0.35869932174682617, "cos_r_negg": [0.23496845364570618, 0.2489742934703827, 0.2390528917312622], "cos_innovation_negg": [0.5499919652938843, 0.5420595407485962, 0.4637102484703064], "cos_apical_negg": [0.23496845364570618, 0.24897430837154388, 0.2390528917312622], "cos_Ac_negg": [0.5499877333641052, 0.5419607162475586, 0.46368440985679626], "r_norm": [0.7770188450813293, 0.76595139503479, 0.7725645899772644]}, {"step": 300, "epoch": 0, "train_loss": 0.25705695152282715, "cos_r_negg": [0.2327309250831604, 0.23788033425807953, 0.25863051414489746], "cos_innovation_negg": [0.5484759211540222, 0.5564261674880981, 0.5521402359008789], "cos_apical_negg": [0.2327309250831604, 0.23788033425807953, 0.25863051414489746], "cos_Ac_negg": [0.5482003092765808, 0.5562231540679932, 0.5521334409713745], "r_norm": [0.6918046474456787, 0.6813380718231201, 0.6911088228225708]}, {"step": 400, "epoch": 0, "train_loss": 0.17998269200325012, "cos_r_negg": [0.2094043791294098, 0.2362065464258194, 0.2659427523612976], "cos_innovation_negg": [0.536241352558136, 0.5778297781944275, 0.5647197365760803], "cos_apical_negg": [0.2094043791294098, 0.2362065464258194, 0.2659427523612976], "cos_Ac_negg": [0.5361621379852295, 0.5775743722915649, 0.5645044445991516], "r_norm": [0.679763913154602, 0.6754298210144043, 0.6804885864257812]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9355, "test_loss": 0.21739756927490234}, {"step": 500, "epoch": 1, "train_loss": 0.2138676792383194, "cos_r_negg": [0.19318073987960815, 0.21462637186050415, 0.24055153131484985], "cos_innovation_negg": [0.5731021165847778, 0.5893297791481018, 0.6107022762298584], "cos_apical_negg": [0.19318073987960815, 0.21462638676166534, 0.24055154621601105], "cos_Ac_negg": [0.5742365121841431, 0.5893971920013428, 0.609838604927063], "r_norm": [0.5675994157791138, 0.561404287815094, 0.5678472518920898], "ldr": {"dL_sdil": -0.01185181736946106, "dL_bp": -0.010675176978111267, "ratio": 1.1102220969040995}}, {"step": 600, "epoch": 1, "train_loss": 0.09671124070882797, "cos_r_negg": [0.16785210371017456, 0.19913065433502197, 0.22777554392814636], "cos_innovation_negg": [0.568167507648468, 0.5897747278213501, 0.6196615695953369], "cos_apical_negg": [0.16785210371017456, 0.19913065433502197, 0.22777554392814636], "cos_Ac_negg": [0.5680513381958008, 0.5894980430603027, 0.6193525791168213], "r_norm": [0.45053526759147644, 0.4490148723125458, 0.4517337679862976]}, {"step": 700, "epoch": 1, "train_loss": 0.25853413343429565, "cos_r_negg": [0.1954101324081421, 0.2096310257911682, 0.2290453165769577], "cos_innovation_negg": [0.5455281734466553, 0.5825574398040771, 0.6364662647247314], "cos_apical_negg": [0.1954101324081421, 0.2096310257911682, 0.2290453165769577], "cos_Ac_negg": [0.5453530550003052, 0.5823148488998413, 0.6363263130187988], "r_norm": [0.452070415019989, 0.4455122947692871, 0.45027536153793335]}, {"step": 800, "epoch": 1, "train_loss": 0.23864632844924927, "cos_r_negg": [0.16706319153308868, 0.18102678656578064, 0.23181748390197754], "cos_innovation_negg": [0.564144492149353, 0.5755118131637573, 0.6405776739120483], "cos_apical_negg": [0.16706319153308868, 0.18102678656578064, 0.23181748390197754], "cos_Ac_negg": [0.5643078088760376, 0.5752726197242737, 0.6404033899307251], "r_norm": [0.42372632026672363, 0.4244443476200104, 0.4254371225833893]}, {"step": 900, "epoch": 1, "train_loss": 0.19433821737766266, "cos_r_negg": [0.19115428626537323, 0.2319030910730362, 0.2743412256240845], "cos_innovation_negg": [0.5572805404663086, 0.5882014036178589, 0.6656894683837891], "cos_apical_negg": [0.19115427136421204, 0.2319030910730362, 0.2743411958217621], "cos_Ac_negg": [0.5571479797363281, 0.5879946351051331, 0.6655135750770569], "r_norm": [0.4814557433128357, 0.4753068685531616, 0.48049071431159973]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9541, "test_loss": 0.16767139129638672}, {"step": 1000, "epoch": 2, "train_loss": 0.07323691993951797, "cos_r_negg": [0.18533377349376678, 0.20593209564685822, 0.22659942507743835], "cos_innovation_negg": [0.5686722993850708, 0.5983743667602539, 0.6596771478652954], "cos_apical_negg": [0.1853337585926056, 0.20593209564685822, 0.22659939527511597], "cos_Ac_negg": [0.5684210062026978, 0.5980938673019409, 0.6594594120979309], "r_norm": [0.48285090923309326, 0.4805031418800354, 0.48812124133110046], "ldr": {"dL_sdil": -0.01850762963294983, "dL_bp": -0.018236294388771057, "ratio": 1.0148788585221484}}, {"step": 1100, "epoch": 2, "train_loss": 0.18410810828208923, "cos_r_negg": [0.15007540583610535, 0.17071875929832458, 0.21554194390773773], "cos_innovation_negg": [0.5643891096115112, 0.614929735660553, 0.7009598016738892], "cos_apical_negg": [0.15007540583610535, 0.17071875929832458, 0.21554194390773773], "cos_Ac_negg": [0.5641417503356934, 0.6147982478141785, 0.7007851600646973], "r_norm": [0.39942309260368347, 0.3952123820781708, 0.399445116519928]}, {"step": 1200, "epoch": 2, "train_loss": 0.1506100744009018, "cos_r_negg": [0.16346396505832672, 0.18483838438987732, 0.21001099050045013], "cos_innovation_negg": [0.5539546012878418, 0.6020072102546692, 0.7045530080795288], "cos_apical_negg": [0.16346396505832672, 0.18483838438987732, 0.21001097559928894], "cos_Ac_negg": [0.5536083579063416, 0.6017486453056335, 0.704311192035675], "r_norm": [0.42719393968582153, 0.4216434955596924, 0.4261261224746704]}, {"step": 1300, "epoch": 2, "train_loss": 0.18525248765945435, "cos_r_negg": [0.1321549415588379, 0.16249878704547882, 0.20184743404388428], "cos_innovation_negg": [0.5685097575187683, 0.6059775948524475, 0.7228296399116516], "cos_apical_negg": [0.1321549415588379, 0.16249877214431763, 0.20184743404388428], "cos_Ac_negg": [0.5683749914169312, 0.6057950854301453, 0.722582221031189], "r_norm": [0.39254075288772583, 0.389163613319397, 0.3909877836704254]}, {"step": 1400, "epoch": 2, "train_loss": 0.15481902658939362, "cos_r_negg": [0.1297166794538498, 0.1661204695701599, 0.2534674406051636], "cos_innovation_negg": [0.5598414540290833, 0.6230447888374329, 0.6980350017547607], "cos_apical_negg": [0.1297166794538498, 0.1661204844713211, 0.25346747040748596], "cos_Ac_negg": [0.5596442222595215, 0.6228131055831909, 0.6977487802505493], "r_norm": [0.3701799511909485, 0.3658253848552704, 0.37039294838905334]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9607, "test_loss": 0.13900364608764648}, {"step": 1500, "epoch": 3, "train_loss": 0.110859215259552, "cos_r_negg": [0.14650633931159973, 0.16383367776870728, 0.23567108809947968], "cos_innovation_negg": [0.575775146484375, 0.6230165362358093, 0.7273306846618652], "cos_apical_negg": [0.14650633931159973, 0.16383367776870728, 0.23567108809947968], "cos_Ac_negg": [0.5751730799674988, 0.6225860714912415, 0.7268973588943481], "r_norm": [0.3419315218925476, 0.336915522813797, 0.340427041053772], "ldr": {"dL_sdil": -0.004863366484642029, "dL_bp": -0.005314543843269348, "ratio": 0.9151051582350351}}, {"step": 1600, "epoch": 3, "train_loss": 0.10772039741277695, "cos_r_negg": [0.14053931832313538, 0.1739019751548767, 0.22011321783065796], "cos_innovation_negg": [0.5879946947097778, 0.6072326302528381, 0.7198115587234497], "cos_apical_negg": [0.14053931832313538, 0.1739019751548767, 0.22011321783065796], "cos_Ac_negg": [0.5874791145324707, 0.6069145202636719, 0.7194622755050659], "r_norm": [0.356960654258728, 0.35426419973373413, 0.3570789694786072]}, {"step": 1700, "epoch": 3, "train_loss": 0.09103884547948837, "cos_r_negg": [0.14236924052238464, 0.1826709359884262, 0.21392950415611267], "cos_innovation_negg": [0.573638916015625, 0.6218717694282532, 0.7085980772972107], "cos_apical_negg": [0.14236922562122345, 0.1826709359884262, 0.21392948925495148], "cos_Ac_negg": [0.5734999179840088, 0.6216661930084229, 0.7083688974380493], "r_norm": [0.3555583357810974, 0.3530515134334564, 0.3562694787979126]}, {"step": 1800, "epoch": 3, "train_loss": 0.030617505311965942, "cos_r_negg": [0.131395623087883, 0.14451764523983002, 0.21224914491176605], "cos_innovation_negg": [0.5636694431304932, 0.6041983962059021, 0.6821848750114441], "cos_apical_negg": [0.131395623087883, 0.14451764523983002, 0.21224914491176605], "cos_Ac_negg": [0.5631300210952759, 0.6032777428627014, 0.6802993416786194], "r_norm": [0.45386093854904175, 0.44351261854171753, 0.45473772287368774]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9606, "test_loss": 0.1345690662384033}, {"step": 1900, "epoch": 4, "train_loss": 0.06669837236404419, "cos_r_negg": [0.12382270395755768, 0.155398428440094, 0.18213950097560883], "cos_innovation_negg": [0.5737964510917664, 0.6262027025222778, 0.7180590629577637], "cos_apical_negg": [0.12382270395755768, 0.155398428440094, 0.18213950097560883], "cos_Ac_negg": [0.5733422040939331, 0.6257920265197754, 0.7177380323410034], "r_norm": [0.2786588966846466, 0.27537527680397034, 0.2776572108268738]}, {"step": 2000, "epoch": 4, "train_loss": 0.11916625499725342, "cos_r_negg": [0.09751120954751968, 0.13077056407928467, 0.14181913435459137], "cos_innovation_negg": [0.5742237567901611, 0.6206357479095459, 0.7055182456970215], "cos_apical_negg": [0.09751120209693909, 0.13077056407928467, 0.14181913435459137], "cos_Ac_negg": [0.5737118124961853, 0.6201224327087402, 0.7049134969711304], "r_norm": [0.2879878282546997, 0.2822757959365845, 0.2883542776107788], "ldr": {"dL_sdil": -0.008462302386760712, "dL_bp": -0.010234296321868896, "ratio": 0.8268572768093744}}, {"step": 2100, "epoch": 4, "train_loss": 0.06574305891990662, "cos_r_negg": [0.1261293590068817, 0.165409117937088, 0.21647286415100098], "cos_innovation_negg": [0.587814211845398, 0.6350489854812622, 0.6933523416519165], "cos_apical_negg": [0.1261293590068817, 0.1654091477394104, 0.21647286415100098], "cos_Ac_negg": [0.5874064564704895, 0.6346099376678467, 0.6926973462104797], "r_norm": [0.3062635362148285, 0.3029603064060211, 0.3067421317100525]}, {"step": 2200, "epoch": 4, "train_loss": 0.050690002739429474, "cos_r_negg": [0.10052382946014404, 0.1354762464761734, 0.21772167086601257], "cos_innovation_negg": [0.5667058825492859, 0.6212449669837952, 0.6726551651954651], "cos_apical_negg": [0.10052382946014404, 0.1354762464761734, 0.21772167086601257], "cos_Ac_negg": [0.5662604570388794, 0.6210074424743652, 0.6722522974014282], "r_norm": [0.3140619993209839, 0.3138364851474762, 0.3157208561897278]}, {"step": 2300, "epoch": 4, "train_loss": 0.2856258749961853, "cos_r_negg": [0.12347837537527084, 0.13504227995872498, 0.1656659096479416], "cos_innovation_negg": [0.5976657867431641, 0.63372802734375, 0.7001543045043945], "cos_apical_negg": [0.12347836792469025, 0.13504226505756378, 0.1656659096479416], "cos_Ac_negg": [0.5968269109725952, 0.6332087516784668, 0.699736475944519], "r_norm": [0.30333665013313293, 0.3017101287841797, 0.3041290044784546]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9656, "test_loss": 0.12579462852478027}, {"step": 2400, "epoch": 5, "train_loss": 0.10922393947839737, "cos_r_negg": [0.0800085961818695, 0.1303296536207199, 0.17170783877372742], "cos_innovation_negg": [0.5716754198074341, 0.6151139140129089, 0.7041137218475342], "cos_apical_negg": [0.0800085961818695, 0.1303296536207199, 0.17170783877372742], "cos_Ac_negg": [0.5712298154830933, 0.6144628524780273, 0.70359206199646], "r_norm": [0.26329895853996277, 0.2613946199417114, 0.26422256231307983]}, {"step": 2500, "epoch": 5, "train_loss": 0.03882201761007309, "cos_r_negg": [0.10450796782970428, 0.14539843797683716, 0.14641329646110535], "cos_innovation_negg": [0.5877382755279541, 0.6312771439552307, 0.7209851741790771], "cos_apical_negg": [0.10450796782970428, 0.14539843797683716, 0.14641329646110535], "cos_Ac_negg": [0.5871776938438416, 0.630534291267395, 0.719475269317627], "r_norm": [0.300679475069046, 0.296906441450119, 0.30106282234191895], "ldr": {"dL_sdil": -0.010640151798725128, "dL_bp": -0.012677699327468872, "ratio": 0.8392809707729089}}, {"step": 2600, "epoch": 5, "train_loss": 0.08008192479610443, "cos_r_negg": [0.11345787346363068, 0.1607760190963745, 0.17814789712429047], "cos_innovation_negg": [0.5683139562606812, 0.615220308303833, 0.723012387752533], "cos_apical_negg": [0.11345787346363068, 0.1607760190963745, 0.17814789712429047], "cos_Ac_negg": [0.5678278207778931, 0.6144174337387085, 0.721819281578064], "r_norm": [0.30044424533843994, 0.29629045724868774, 0.3002690374851227]}, {"step": 2700, "epoch": 5, "train_loss": 0.060378801077604294, "cos_r_negg": [0.0910666212439537, 0.1561325490474701, 0.17633703351020813], "cos_innovation_negg": [0.5736572742462158, 0.6276485323905945, 0.7415227890014648], "cos_apical_negg": [0.0910666212439537, 0.1561325192451477, 0.17633703351020813], "cos_Ac_negg": [0.5730524063110352, 0.6272779703140259, 0.740976870059967], "r_norm": [0.2680075168609619, 0.2659075856208801, 0.2676078677177429]}, {"step": 2800, "epoch": 5, "train_loss": 0.13768145442008972, "cos_r_negg": [0.09511639177799225, 0.1233239695429802, 0.15861858427524567], "cos_innovation_negg": [0.5799553394317627, 0.6164218187332153, 0.7362614870071411], "cos_apical_negg": [0.09511638432741165, 0.12332397699356079, 0.15861856937408447], "cos_Ac_negg": [0.5797080993652344, 0.6170012950897217, 0.7352924346923828], "r_norm": [0.2905276119709015, 0.2830788493156433, 0.28721317648887634]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.965, "test_loss": 0.1283594997406006}, {"step": 2900, "epoch": 6, "train_loss": 0.05049640312790871, "cos_r_negg": [0.08567244559526443, 0.10055835545063019, 0.1446242332458496], "cos_innovation_negg": [0.5851818323135376, 0.640073299407959, 0.7310131788253784], "cos_apical_negg": [0.08567244559526443, 0.10055836290121078, 0.1446242481470108], "cos_Ac_negg": [0.5852477550506592, 0.6404622793197632, 0.731020987033844], "r_norm": [0.29810234904289246, 0.2922148108482361, 0.29742997884750366]}, {"step": 3000, "epoch": 6, "train_loss": 0.13612395524978638, "cos_r_negg": [0.06678971648216248, 0.09917480498552322, 0.10427999496459961], "cos_innovation_negg": [0.5583495497703552, 0.6143648624420166, 0.7150453925132751], "cos_apical_negg": [0.06678971648216248, 0.09917480498552322, 0.10427998751401901], "cos_Ac_negg": [0.5581167936325073, 0.6140143871307373, 0.7146580219268799], "r_norm": [0.233027845621109, 0.22977158427238464, 0.23344995081424713], "ldr": {"dL_sdil": -0.004542052745819092, "dL_bp": -0.006951414048671722, "ratio": 0.653399828296372}}, {"step": 3100, "epoch": 6, "train_loss": 0.031174927949905396, "cos_r_negg": [0.07337063550949097, 0.11348855495452881, 0.1354759931564331], "cos_innovation_negg": [0.5577149391174316, 0.6143739819526672, 0.719869077205658], "cos_apical_negg": [0.07337063550949097, 0.11348855495452881, 0.1354759931564331], "cos_Ac_negg": [0.5575193762779236, 0.6142213344573975, 0.7192609310150146], "r_norm": [0.26552560925483704, 0.26270338892936707, 0.2663534879684448]}, {"step": 3200, "epoch": 6, "train_loss": 0.058665502816438675, "cos_r_negg": [0.07464861869812012, 0.10516990721225739, 0.12432101368904114], "cos_innovation_negg": [0.5637915134429932, 0.6214838027954102, 0.7110974788665771], "cos_apical_negg": [0.07464861869812012, 0.10516990721225739, 0.12432101368904114], "cos_Ac_negg": [0.5637941360473633, 0.6208317279815674, 0.7090954780578613], "r_norm": [0.221273273229599, 0.21947352588176727, 0.22070924937725067]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9643, "test_loss": 0.13263772964477538}, {"step": 3300, "epoch": 7, "train_loss": 0.019306685775518417, "cos_r_negg": [0.07242114841938019, 0.09392867237329483, 0.08804640173912048], "cos_innovation_negg": [0.5655518770217896, 0.6146605014801025, 0.710368275642395], "cos_apical_negg": [0.07242114841938019, 0.09392867982387543, 0.08804639428853989], "cos_Ac_negg": [0.5656253695487976, 0.6150987148284912, 0.7095399498939514], "r_norm": [0.2598894238471985, 0.25448179244995117, 0.2578650116920471]}, {"step": 3400, "epoch": 7, "train_loss": 0.0447990819811821, "cos_r_negg": [0.0849866047501564, 0.0959339588880539, 0.12536758184432983], "cos_innovation_negg": [0.5790256857872009, 0.6278877258300781, 0.743285059928894], "cos_apical_negg": [0.0849865972995758, 0.0959339588880539, 0.12536758184432983], "cos_Ac_negg": [0.5787602663040161, 0.6278969049453735, 0.7430012226104736], "r_norm": [0.22801300883293152, 0.22419679164886475, 0.2248200923204422]}, {"step": 3500, "epoch": 7, "train_loss": 0.029470354318618774, "cos_r_negg": [0.0645921528339386, 0.08396442234516144, 0.07825030386447906], "cos_innovation_negg": [0.5549382567405701, 0.6131943464279175, 0.7136404514312744], "cos_apical_negg": [0.0645921528339386, 0.08396442234516144, 0.07825030386447906], "cos_Ac_negg": [0.5542522072792053, 0.6133655905723572, 0.712344229221344], "r_norm": [0.21154215931892395, 0.20973695814609528, 0.21197417378425598], "ldr": {"dL_sdil": -0.0032623186707496643, "dL_bp": -0.0049062371253967285, "ratio": 0.6649329389039398}}, {"step": 3600, "epoch": 7, "train_loss": 0.017343800514936447, "cos_r_negg": [0.07564590871334076, 0.09254085272550583, 0.09874577075242996], "cos_innovation_negg": [0.5643134117126465, 0.6200522184371948, 0.7193397283554077], "cos_apical_negg": [0.07564590871334076, 0.09254084527492523, 0.09874577820301056], "cos_Ac_negg": [0.5647028088569641, 0.6200195550918579, 0.717861533164978], "r_norm": [0.24469712376594543, 0.24186371266841888, 0.24495328962802887]}, {"step": 3700, "epoch": 7, "train_loss": 0.02560429461300373, "cos_r_negg": [0.0934041291475296, 0.10227875411510468, 0.11355741322040558], "cos_innovation_negg": [0.5772973299026489, 0.6290334463119507, 0.7404904961585999], "cos_apical_negg": [0.0934041291475296, 0.10227875411510468, 0.11355740576982498], "cos_Ac_negg": [0.5771619081497192, 0.6288613080978394, 0.7394621968269348], "r_norm": [0.235325887799263, 0.23235464096069336, 0.23532480001449585]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9669, "test_loss": 0.12723760547637938}, {"step": 3800, "epoch": 8, "train_loss": 0.02622942626476288, "cos_r_negg": [0.04437864199280739, 0.07326246052980423, 0.08763329684734344], "cos_innovation_negg": [0.561237633228302, 0.6289453506469727, 0.7156274318695068], "cos_apical_negg": [0.04437864571809769, 0.07326246052980423, 0.08763329684734344], "cos_Ac_negg": [0.5624828934669495, 0.6301823258399963, 0.7157561182975769], "r_norm": [0.21577385067939758, 0.21152213215827942, 0.21474748849868774]}, {"step": 3900, "epoch": 8, "train_loss": 0.11024263501167297, "cos_r_negg": [0.07163235545158386, 0.08981633186340332, 0.10607653856277466], "cos_innovation_negg": [0.5642953515052795, 0.6160035133361816, 0.6934971809387207], "cos_apical_negg": [0.07163235545158386, 0.08981633186340332, 0.10607653856277466], "cos_Ac_negg": [0.5641449689865112, 0.616386890411377, 0.692419171333313], "r_norm": [0.2235879898071289, 0.22061657905578613, 0.22167834639549255]}, {"step": 4000, "epoch": 8, "train_loss": 0.032469671219587326, "cos_r_negg": [0.05761587619781494, 0.08533866703510284, 0.10965381562709808], "cos_innovation_negg": [0.5626274347305298, 0.629766047000885, 0.7038819789886475], "cos_apical_negg": [0.05761587619781494, 0.08533865958452225, 0.10965380817651749], "cos_Ac_negg": [0.5637836456298828, 0.6311998963356018, 0.7026136517524719], "r_norm": [0.2095927894115448, 0.20795543491840363, 0.2076958417892456], "ldr": {"dL_sdil": -0.004674933850765228, "dL_bp": -0.00748114287853241, "ratio": 0.6248956779291347}}, {"step": 4100, "epoch": 8, "train_loss": 0.015821238979697227, "cos_r_negg": [0.06512697786092758, 0.06940074265003204, 0.10688220709562302], "cos_innovation_negg": [0.5558160543441772, 0.6281419992446899, 0.7058539390563965], "cos_apical_negg": [0.06512698531150818, 0.06940074265003204, 0.10688220709562302], "cos_Ac_negg": [0.5582858920097351, 0.6296072006225586, 0.7058401107788086], "r_norm": [0.23751769959926605, 0.23330527544021606, 0.23546218872070312]}, {"step": 4200, "epoch": 8, "train_loss": 0.0786832720041275, "cos_r_negg": [0.07665185630321503, 0.07518652081489563, 0.12023349851369858], "cos_innovation_negg": [0.5658077001571655, 0.6285502314567566, 0.7334243059158325], "cos_apical_negg": [0.07665185630321503, 0.07518652081489563, 0.12023349106311798], "cos_Ac_negg": [0.5689389705657959, 0.6302039623260498, 0.7326260209083557], "r_norm": [0.17267274856567383, 0.1692860722541809, 0.17201559245586395]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9699, "test_loss": 0.12432236766815186}, {"step": 4300, "epoch": 9, "train_loss": 0.03487960249185562, "cos_r_negg": [0.04885092377662659, 0.09549008309841156, 0.14046651124954224], "cos_innovation_negg": [0.5548266172409058, 0.6173526048660278, 0.7183548212051392], "cos_apical_negg": [0.04885092377662659, 0.09549008309841156, 0.14046651124954224], "cos_Ac_negg": [0.5621811747550964, 0.6217618584632874, 0.7210865020751953], "r_norm": [0.22707624733448029, 0.22213496267795563, 0.22637689113616943]}, {"step": 4400, "epoch": 9, "train_loss": 0.016691552475094795, "cos_r_negg": [0.04015784710645676, 0.09020178020000458, 0.09162594377994537], "cos_innovation_negg": [0.5598901510238647, 0.6221605539321899, 0.73450767993927], "cos_apical_negg": [0.04015784338116646, 0.09020177274942398, 0.09162594377994537], "cos_Ac_negg": [0.5633233785629272, 0.6249106526374817, 0.7337782382965088], "r_norm": [0.21741098165512085, 0.21283850073814392, 0.21458357572555542]}, {"step": 4500, "epoch": 9, "train_loss": 0.08103659003973007, "cos_r_negg": [0.07006952166557312, 0.09883368760347366, 0.10027097910642624], "cos_innovation_negg": [0.5810632705688477, 0.64183509349823, 0.7424862384796143], "cos_apical_negg": [0.07006952166557312, 0.09883368015289307, 0.10027097165584564], "cos_Ac_negg": [0.5830268859863281, 0.64426589012146, 0.7422841191291809], "r_norm": [0.23907096683979034, 0.23545652627944946, 0.23635035753250122], "ldr": {"dL_sdil": -0.0072174072265625, "dL_bp": -0.012745216488838196, "ratio": 0.5662836118070844}}, {"step": 4600, "epoch": 9, "train_loss": 0.01189473643898964, "cos_r_negg": [0.04925631359219551, 0.09685302525758743, 0.09273850172758102], "cos_innovation_negg": [0.5442144870758057, 0.6160503625869751, 0.7281934022903442], "cos_apical_negg": [0.04925630986690521, 0.09685303270816803, 0.09273849427700043], "cos_Ac_negg": [0.549444317817688, 0.6212999820709229, 0.7269050478935242], "r_norm": [0.22633762657642365, 0.2216828614473343, 0.22389601171016693]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9705, "test_loss": 0.1226830888748169}, {"step": 4700, "epoch": 10, "train_loss": 0.01758262701332569, "cos_r_negg": [0.049477897584438324, 0.05030638724565506, 0.06188071519136429], "cos_innovation_negg": [0.5504025220870972, 0.6181167960166931, 0.7068608999252319], "cos_apical_negg": [0.049477897584438324, 0.05030638724565506, 0.06188071891665459], "cos_Ac_negg": [0.5557631254196167, 0.6228522658348083, 0.7075021266937256], "r_norm": [0.20983056724071503, 0.20785710215568542, 0.2090214639902115]}, {"step": 4800, "epoch": 10, "train_loss": 0.06082163378596306, "cos_r_negg": [0.030915986746549606, 0.05864178016781807, 0.08276590704917908], "cos_innovation_negg": [0.5669792890548706, 0.6159775257110596, 0.7320576906204224], "cos_apical_negg": [0.030915983021259308, 0.05864178389310837, 0.08276590704917908], "cos_Ac_negg": [0.5709697604179382, 0.6198879480361938, 0.7328310012817383], "r_norm": [0.2022201120853424, 0.19934101402759552, 0.20066407322883606]}, {"step": 4900, "epoch": 10, "train_loss": 0.032953646034002304, "cos_r_negg": [0.05380989611148834, 0.08456790447235107, 0.08466912806034088], "cos_innovation_negg": [0.5723910927772522, 0.6243938207626343, 0.7248529195785522], "cos_apical_negg": [0.05380989611148834, 0.08456790447235107, 0.08466912806034088], "cos_Ac_negg": [0.5737979412078857, 0.627792477607727, 0.7235276699066162], "r_norm": [0.21145141124725342, 0.207720547914505, 0.20969641208648682]}, {"step": 5000, "epoch": 10, "train_loss": 0.008504677563905716, "cos_r_negg": [0.06354457139968872, 0.07587283104658127, 0.06342294812202454], "cos_innovation_negg": [0.5497069358825684, 0.6113346219062805, 0.7288207411766052], "cos_apical_negg": [0.06354457139968872, 0.07587282359600067, 0.06342294812202454], "cos_Ac_negg": [0.5550521016120911, 0.6166893243789673, 0.7290618419647217], "r_norm": [0.18436767160892487, 0.1815374195575714, 0.18342182040214539], "ldr": {"dL_sdil": -0.0031595155596733093, "dL_bp": -0.006094314157962799, "ratio": 0.5184366079233219}}, {"step": 5100, "epoch": 10, "train_loss": 0.051396749913692474, "cos_r_negg": [0.06187586486339569, 0.07489577680826187, 0.10692720860242844], "cos_innovation_negg": [0.5524444580078125, 0.6378144025802612, 0.7474163174629211], "cos_apical_negg": [0.06187586486339569, 0.07489577680826187, 0.10692720860242844], "cos_Ac_negg": [0.5572330355644226, 0.6413361430168152, 0.7495434284210205], "r_norm": [0.19959013164043427, 0.1974097490310669, 0.1995234191417694]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9692, "test_loss": 0.1310543426513672}, {"step": 5200, "epoch": 11, "train_loss": 0.007362754084169865, "cos_r_negg": [0.0649820864200592, 0.08390449732542038, 0.09689496457576752], "cos_innovation_negg": [0.5592031478881836, 0.6397548913955688, 0.7427685856819153], "cos_apical_negg": [0.06498207896947861, 0.08390449732542038, 0.09689496457576752], "cos_Ac_negg": [0.5643541812896729, 0.6436315178871155, 0.7445123195648193], "r_norm": [0.22553887963294983, 0.2213938981294632, 0.2233155220746994]}, {"step": 5300, "epoch": 11, "train_loss": 0.011183683760464191, "cos_r_negg": [0.043541137129068375, 0.06899329274892807, 0.10155381262302399], "cos_innovation_negg": [0.5573720932006836, 0.6282849311828613, 0.7575475573539734], "cos_apical_negg": [0.043541137129068375, 0.06899329274892807, 0.10155382007360458], "cos_Ac_negg": [0.5621886253356934, 0.6333240270614624, 0.7567877769470215], "r_norm": [0.18876740336418152, 0.18407520651817322, 0.18542222678661346]}, {"step": 5400, "epoch": 11, "train_loss": 0.011638451367616653, "cos_r_negg": [0.04954775795340538, 0.059115152806043625, 0.08713265508413315], "cos_innovation_negg": [0.5465665459632874, 0.6171325445175171, 0.7398940324783325], "cos_apical_negg": [0.04954775422811508, 0.059115152806043625, 0.08713264763355255], "cos_Ac_negg": [0.5519935488700867, 0.6228265762329102, 0.7411057353019714], "r_norm": [0.19537073373794556, 0.1906038224697113, 0.19320827722549438]}, {"step": 5500, "epoch": 11, "train_loss": 0.02806015871465206, "cos_r_negg": [0.06098166108131409, 0.062265340238809586, 0.09589625149965286], "cos_innovation_negg": [0.5488113760948181, 0.6229140758514404, 0.7256801724433899], "cos_apical_negg": [0.06098165735602379, 0.062265340238809586, 0.09589624404907227], "cos_Ac_negg": [0.5533926486968994, 0.6290234327316284, 0.7265867590904236], "r_norm": [0.20373515784740448, 0.20048436522483826, 0.20147669315338135], "ldr": {"dL_sdil": -0.004074700176715851, "dL_bp": -0.00852278620004654, "ratio": 0.4780948484538543}}, {"step": 5600, "epoch": 11, "train_loss": 0.010782569646835327, "cos_r_negg": [0.049422577023506165, 0.06817072629928589, 0.12336263805627823], "cos_innovation_negg": [0.5475457906723022, 0.6164689064025879, 0.7429060935974121], "cos_apical_negg": [0.049422577023506165, 0.06817072629928589, 0.12336264550685883], "cos_Ac_negg": [0.5579742193222046, 0.6262809634208679, 0.746960461139679], "r_norm": [0.2047860473394394, 0.20101822912693024, 0.20428185164928436]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9706, "test_loss": 0.12458508834838868}, {"step": 5700, "epoch": 12, "train_loss": 0.012705364264547825, "cos_r_negg": [0.04466864466667175, 0.07217530906200409, 0.12809297442436218], "cos_innovation_negg": [0.5514708757400513, 0.6251588463783264, 0.7614907622337341], "cos_apical_negg": [0.04466864466667175, 0.07217530906200409, 0.12809298932552338], "cos_Ac_negg": [0.5596020221710205, 0.6324319243431091, 0.7608262896537781], "r_norm": [0.22649411857128143, 0.22218579053878784, 0.2237890660762787]}, {"step": 5800, "epoch": 12, "train_loss": 0.007963050156831741, "cos_r_negg": [0.04596461355686188, 0.05485370382666588, 0.09964601695537567], "cos_innovation_negg": [0.5427035093307495, 0.6207987070083618, 0.7493276000022888], "cos_apical_negg": [0.04596461355686188, 0.05485369265079498, 0.09964600950479507], "cos_Ac_negg": [0.5507823824882507, 0.6290920376777649, 0.7533329725265503], "r_norm": [0.1967996209859848, 0.1934458613395691, 0.19505123794078827]}, {"step": 5900, "epoch": 12, "train_loss": 0.005631913430988789, "cos_r_negg": [0.02647482231259346, 0.05131883919239044, 0.11191203445196152], "cos_innovation_negg": [0.5246216058731079, 0.597536563873291, 0.7366989850997925], "cos_apical_negg": [0.02647482231259346, 0.05131884291768074, 0.11191204190254211], "cos_Ac_negg": [0.5407565832138062, 0.6185312867164612, 0.7418075799942017], "r_norm": [0.2164660096168518, 0.21117088198661804, 0.21352757513523102]}, {"step": 6000, "epoch": 12, "train_loss": 0.007956218905746937, "cos_r_negg": [0.03920429199934006, 0.0754752978682518, 0.12268101423978806], "cos_innovation_negg": [0.5336796045303345, 0.6201992630958557, 0.7478694915771484], "cos_apical_negg": [0.03920429199934006, 0.0754752904176712, 0.12268102914094925], "cos_Ac_negg": [0.5447767376899719, 0.6308369636535645, 0.7444664239883423], "r_norm": [0.21638329327106476, 0.21274390816688538, 0.21485933661460876], "ldr": {"dL_sdil": -0.007918410003185272, "dL_bp": -0.016881465911865234, "ratio": 0.46905938409174364}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.967, "test_loss": 0.13685971374511718}, {"step": 6100, "epoch": 13, "train_loss": 0.02517504058778286, "cos_r_negg": [0.056664321571588516, 0.06747549027204514, 0.09756827354431152], "cos_innovation_negg": [0.5624748468399048, 0.6267976760864258, 0.765487790107727], "cos_apical_negg": [0.056664325296878815, 0.06747548282146454, 0.09756826609373093], "cos_Ac_negg": [0.5680994987487793, 0.6344419717788696, 0.7677033543586731], "r_norm": [0.19518572092056274, 0.19355317950248718, 0.19456177949905396]}, {"step": 6200, "epoch": 13, "train_loss": 0.012008452787995338, "cos_r_negg": [0.02989443950355053, 0.059632353484630585, 0.09686162322759628], "cos_innovation_negg": [0.5549407601356506, 0.6254200339317322, 0.7544149160385132], "cos_apical_negg": [0.029894443228840828, 0.059632353484630585, 0.09686162322759628], "cos_Ac_negg": [0.5630207657814026, 0.6364709138870239, 0.7583058476448059], "r_norm": [0.18568243086338043, 0.18361473083496094, 0.18413721024990082]}, {"step": 6300, "epoch": 13, "train_loss": 0.00748961279168725, "cos_r_negg": [0.04999706149101257, 0.07078620791435242, 0.10934516042470932], "cos_innovation_negg": [0.5585170984268188, 0.6274136900901794, 0.7738465070724487], "cos_apical_negg": [0.04999706149101257, 0.07078620791435242, 0.10934515297412872], "cos_Ac_negg": [0.5641648173332214, 0.6356494426727295, 0.7718241214752197], "r_norm": [0.2185528725385666, 0.21450120210647583, 0.21682244539260864]}, {"step": 6400, "epoch": 13, "train_loss": 0.0045163496397435665, "cos_r_negg": [0.045995961874723434, 0.06127827614545822, 0.10519485920667648], "cos_innovation_negg": [0.5285505652427673, 0.619375467300415, 0.7469182014465332], "cos_apical_negg": [0.04599596560001373, 0.06127827614545822, 0.10519486665725708], "cos_Ac_negg": [0.5412546396255493, 0.6326603889465332, 0.7492698431015015], "r_norm": [0.2200552523136139, 0.2154013067483902, 0.21812433004379272]}, {"step": 6500, "epoch": 13, "train_loss": 0.003596516326069832, "cos_r_negg": [0.04465258866548538, 0.07837079465389252, 0.0702044665813446], "cos_innovation_negg": [0.5400137901306152, 0.6217684745788574, 0.7461681365966797], "cos_apical_negg": [0.04465258866548538, 0.07837080955505371, 0.0702044665813446], "cos_Ac_negg": [0.5481486320495605, 0.6315551996231079, 0.7488528490066528], "r_norm": [0.19117380678653717, 0.18793326616287231, 0.1886856108903885], "ldr": {"dL_sdil": -0.004521757364273071, "dL_bp": -0.011233538389205933, "ratio": 0.402522981415894}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9708, "test_loss": 0.1253721086502075}, {"step": 6600, "epoch": 14, "train_loss": 0.0057099200785160065, "cos_r_negg": [0.047673728317022324, 0.07365186512470245, 0.11604772508144379], "cos_innovation_negg": [0.5477915406227112, 0.6239545941352844, 0.7583589553833008], "cos_apical_negg": [0.047673724591732025, 0.07365186512470245, 0.11604771018028259], "cos_Ac_negg": [0.5555763840675354, 0.6343176364898682, 0.756290853023529], "r_norm": [0.19733330607414246, 0.1941097229719162, 0.19474002718925476]}, {"step": 6700, "epoch": 14, "train_loss": 0.009178523905575275, "cos_r_negg": [0.040871068835258484, 0.06444497406482697, 0.12502259016036987], "cos_innovation_negg": [0.5270074605941772, 0.6034623384475708, 0.7457575798034668], "cos_apical_negg": [0.040871068835258484, 0.06444497406482697, 0.12502259016036987], "cos_Ac_negg": [0.5431610345840454, 0.623917281627655, 0.7431623339653015], "r_norm": [0.21842065453529358, 0.21405012905597687, 0.21624219417572021]}, {"step": 6800, "epoch": 14, "train_loss": 0.024552559480071068, "cos_r_negg": [0.058451589196920395, 0.07176758348941803, 0.09812598675489426], "cos_innovation_negg": [0.5318054556846619, 0.603507936000824, 0.7431598901748657], "cos_apical_negg": [0.058451589196920395, 0.07176758348941803, 0.09812598675489426], "cos_Ac_negg": [0.5433749556541443, 0.6183546781539917, 0.7466729879379272], "r_norm": [0.2067817896604538, 0.20290452241897583, 0.20498932898044586]}, {"step": 6900, "epoch": 14, "train_loss": 0.0077741569839417934, "cos_r_negg": [0.04254547506570816, 0.06389213353395462, 0.08488097786903381], "cos_innovation_negg": [0.5364390015602112, 0.597833514213562, 0.7362339496612549], "cos_apical_negg": [0.04254548251628876, 0.06389213353395462, 0.08488097786903381], "cos_Ac_negg": [0.5489690899848938, 0.6135779023170471, 0.7435045838356018], "r_norm": [0.20058926939964294, 0.19730912148952484, 0.19768646359443665]}, {"step": 7000, "epoch": 14, "train_loss": 0.006662845145910978, "cos_r_negg": [0.05107412114739418, 0.06778308749198914, 0.10127559304237366], "cos_innovation_negg": [0.5472577810287476, 0.6209309697151184, 0.7657016515731812], "cos_apical_negg": [0.05107412114739418, 0.06778308004140854, 0.10127559304237366], "cos_Ac_negg": [0.5561801791191101, 0.634848952293396, 0.7695335149765015], "r_norm": [0.24305014312267303, 0.23800106346607208, 0.24165913462638855], "ldr": {"dL_sdil": -0.006982788443565369, "dL_bp": -0.01557321846485138, "ratio": 0.4483844145207018}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9705, "test_loss": 0.13234205627441406}], "final": {"test_acc": 0.9705, "test_loss": 0.13234205627441406, "wall_s": 25.417935609817505, "cos_r_negg": [0.046488724648952484, 0.06853098422288895, 0.12159444391727448], "cos_innovation_negg": [0.5427753925323486, 0.6186004877090454, 0.7642629146575928], "cos_apical_negg": [0.046488720923662186, 0.06853098422288895, 0.12159444391727448], "cos_Ac_negg": [0.5519708395004272, 0.6351718306541443, 0.771507978439331]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_matched_s4.json b/results/nuis_ctrl_v1_rho0p05_matched_s4.json new file mode 100644 index 0000000..e328b34 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_matched_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_matched_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.012671537697315216, 0.02800951711833477, 0.007870818488299847], "cos_innovation_negg": [-0.012173660099506378, 0.02864690124988556, 0.008688691072165966], "cos_apical_negg": [-0.012671536765992641, 0.02800951898097992, 0.007870818488299847], "cos_Ac_negg": [-0.012173829600214958, 0.02864680252969265, 0.008688618429005146], "r_norm": [4.851627349853516, 4.857475280761719, 4.950671672821045], "ldr": {"dL_sdil": -0.0035965442657470703, "dL_bp": -0.2647535800933838, "ratio": 0.013584497193497812}}, {"step": 100, "epoch": 0, "train_loss": 0.23219233751296997, "cos_r_negg": [0.2754588723182678, 0.2636558711528778, 0.23765763640403748], "cos_innovation_negg": [0.5037184953689575, 0.452640563249588, 0.37586110830307007], "cos_apical_negg": [0.2754588723182678, 0.2636558711528778, 0.23765762150287628], "cos_Ac_negg": [0.5040225386619568, 0.4528741240501404, 0.3758656978607178], "r_norm": [0.9834369421005249, 0.9961608648300171, 1.0172667503356934]}, {"step": 200, "epoch": 0, "train_loss": 0.34803467988967896, "cos_r_negg": [0.25087565183639526, 0.24470993876457214, 0.256639301776886], "cos_innovation_negg": [0.5748940706253052, 0.5144675374031067, 0.4899575114250183], "cos_apical_negg": [0.25087568163871765, 0.24470993876457214, 0.256639301776886], "cos_Ac_negg": [0.5748316049575806, 0.5151128768920898, 0.48962900042533875], "r_norm": [0.7990931272506714, 0.8062308430671692, 0.824716329574585]}, {"step": 300, "epoch": 0, "train_loss": 0.2601604461669922, "cos_r_negg": [0.23714163899421692, 0.23437903821468353, 0.24478471279144287], "cos_innovation_negg": [0.5860227942466736, 0.5364187955856323, 0.5022217631340027], "cos_apical_negg": [0.23714163899421692, 0.23437903821468353, 0.24478471279144287], "cos_Ac_negg": [0.5860090255737305, 0.5364776253700256, 0.501967191696167], "r_norm": [0.6870736479759216, 0.6940897107124329, 0.7081460952758789]}, {"step": 400, "epoch": 0, "train_loss": 0.17238090932369232, "cos_r_negg": [0.2515743672847748, 0.22900241613388062, 0.29884570837020874], "cos_innovation_negg": [0.5916097164154053, 0.5700705051422119, 0.6054179668426514], "cos_apical_negg": [0.2515743672847748, 0.2290024310350418, 0.29884573817253113], "cos_Ac_negg": [0.5913017988204956, 0.5701278448104858, 0.6051889657974243], "r_norm": [0.704098105430603, 0.7122935056686401, 0.7298505902290344]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9396, "test_loss": 0.1992989372253418}, {"step": 500, "epoch": 1, "train_loss": 0.15319669246673584, "cos_r_negg": [0.21580593287944794, 0.20438015460968018, 0.2745811939239502], "cos_innovation_negg": [0.6003937721252441, 0.5664274096488953, 0.6349955201148987], "cos_apical_negg": [0.21580591797828674, 0.20438015460968018, 0.2745811939239502], "cos_Ac_negg": [0.6002517938613892, 0.5664647221565247, 0.6343942284584045], "r_norm": [0.6052091121673584, 0.6105285882949829, 0.626365602016449], "ldr": {"dL_sdil": -0.018771380186080933, "dL_bp": -0.017141401767730713, "ratio": 1.0950901472608099}}, {"step": 600, "epoch": 1, "train_loss": 0.07864898443222046, "cos_r_negg": [0.20271889865398407, 0.18981502950191498, 0.24691298604011536], "cos_innovation_negg": [0.592519223690033, 0.5746285915374756, 0.6541896462440491], "cos_apical_negg": [0.20271891355514526, 0.18981501460075378, 0.24691300094127655], "cos_Ac_negg": [0.5928087830543518, 0.5748134851455688, 0.6538126468658447], "r_norm": [0.492899626493454, 0.49828222393989563, 0.5072487592697144]}, {"step": 700, "epoch": 1, "train_loss": 0.18591119349002838, "cos_r_negg": [0.18774902820587158, 0.19730904698371887, 0.2588205337524414], "cos_innovation_negg": [0.6056630611419678, 0.5952243804931641, 0.672261655330658], "cos_apical_negg": [0.18774902820587158, 0.19730904698371887, 0.2588205337524414], "cos_Ac_negg": [0.6054359674453735, 0.5952709317207336, 0.6717851161956787], "r_norm": [0.4422515034675598, 0.44817012548446655, 0.45675426721572876]}, {"step": 800, "epoch": 1, "train_loss": 0.1314144730567932, "cos_r_negg": [0.1614469736814499, 0.17120105028152466, 0.21990682184696198], "cos_innovation_negg": [0.6020568609237671, 0.5962784290313721, 0.6884433031082153], "cos_apical_negg": [0.1614469736814499, 0.17120105028152466, 0.21990682184696198], "cos_Ac_negg": [0.6013954281806946, 0.5960206389427185, 0.6878578662872314], "r_norm": [0.46227699518203735, 0.46730998158454895, 0.4775695502758026]}, {"step": 900, "epoch": 1, "train_loss": 0.20942148566246033, "cos_r_negg": [0.23213741183280945, 0.21502667665481567, 0.2748867869377136], "cos_innovation_negg": [0.6047796010971069, 0.6024096608161926, 0.6937699317932129], "cos_apical_negg": [0.23213741183280945, 0.21502667665481567, 0.2748867869377136], "cos_Ac_negg": [0.6044625043869019, 0.6022371649742126, 0.693509042263031], "r_norm": [0.4881097674369812, 0.4923962652683258, 0.501143217086792]}, {"epoch_end": 1, "step": 938, "test_acc": 0.954, "test_loss": 0.16249057998657226}, {"step": 1000, "epoch": 2, "train_loss": 0.10555034875869751, "cos_r_negg": [0.19213074445724487, 0.20364949107170105, 0.2541934847831726], "cos_innovation_negg": [0.6145250201225281, 0.5941904783248901, 0.7238161563873291], "cos_apical_negg": [0.19213074445724487, 0.20364949107170105, 0.2541934847831726], "cos_Ac_negg": [0.6137326955795288, 0.5937837362289429, 0.7228925824165344], "r_norm": [0.5051816701889038, 0.5123348236083984, 0.5238111615180969], "ldr": {"dL_sdil": -0.020879611372947693, "dL_bp": -0.02171263098716736, "ratio": 0.9616343309702081}}, {"step": 1100, "epoch": 2, "train_loss": 0.14995025098323822, "cos_r_negg": [0.1608036309480667, 0.17839692533016205, 0.21034549176692963], "cos_innovation_negg": [0.6271437406539917, 0.5999051928520203, 0.7224013209342957], "cos_apical_negg": [0.1608036458492279, 0.17839692533016205, 0.21034547686576843], "cos_Ac_negg": [0.6265978813171387, 0.5995633602142334, 0.7214943170547485], "r_norm": [0.44330960512161255, 0.4478446841239929, 0.45712190866470337]}, {"step": 1200, "epoch": 2, "train_loss": 0.19789744913578033, "cos_r_negg": [0.14214375615119934, 0.16785132884979248, 0.22647646069526672], "cos_innovation_negg": [0.610224187374115, 0.5990985631942749, 0.7230522632598877], "cos_apical_negg": [0.14214375615119934, 0.16785132884979248, 0.22647646069526672], "cos_Ac_negg": [0.6099343299865723, 0.5987405180931091, 0.7220958471298218], "r_norm": [0.39994731545448303, 0.40427687764167786, 0.4147335886955261]}, {"step": 1300, "epoch": 2, "train_loss": 0.16238901019096375, "cos_r_negg": [0.14891840517520905, 0.1672055572271347, 0.1679675281047821], "cos_innovation_negg": [0.6169491410255432, 0.6053744554519653, 0.7373546957969666], "cos_apical_negg": [0.14891840517520905, 0.1672055572271347, 0.1679675281047821], "cos_Ac_negg": [0.6169780492782593, 0.6052446365356445, 0.736665666103363], "r_norm": [0.3367439806461334, 0.3369835615158081, 0.34749698638916016]}, {"step": 1400, "epoch": 2, "train_loss": 0.18364419043064117, "cos_r_negg": [0.18002033233642578, 0.19296342134475708, 0.20677827298641205], "cos_innovation_negg": [0.6146457195281982, 0.6046367883682251, 0.7352592945098877], "cos_apical_negg": [0.18002034723758698, 0.19296342134475708, 0.20677828788757324], "cos_Ac_negg": [0.6141279935836792, 0.6044787168502808, 0.7345358729362488], "r_norm": [0.35159727931022644, 0.3547499179840088, 0.3656826615333557]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9587, "test_loss": 0.14690606803894043}, {"step": 1500, "epoch": 3, "train_loss": 0.08449388295412064, "cos_r_negg": [0.16875985264778137, 0.19142822921276093, 0.19000297784805298], "cos_innovation_negg": [0.610733687877655, 0.6070719957351685, 0.705892026424408], "cos_apical_negg": [0.16875985264778137, 0.19142822921276093, 0.19000299274921417], "cos_Ac_negg": [0.6102322340011597, 0.6068242192268372, 0.7044358849525452], "r_norm": [0.366945743560791, 0.36736220121383667, 0.37842056155204773], "ldr": {"dL_sdil": -0.00750105082988739, "dL_bp": -0.008349701762199402, "ratio": 0.8983615275752714}}, {"step": 1600, "epoch": 3, "train_loss": 0.12243598699569702, "cos_r_negg": [0.15443696081638336, 0.1707608848810196, 0.1941714882850647], "cos_innovation_negg": [0.61644047498703, 0.6026446223258972, 0.7239662408828735], "cos_apical_negg": [0.15443696081638336, 0.1707608848810196, 0.1941714733839035], "cos_Ac_negg": [0.6157453060150146, 0.6021163463592529, 0.7227789163589478], "r_norm": [0.36821186542510986, 0.3698063790798187, 0.38126057386398315]}, {"step": 1700, "epoch": 3, "train_loss": 0.06708086282014847, "cos_r_negg": [0.1473718285560608, 0.14795580506324768, 0.143108069896698], "cos_innovation_negg": [0.5790796279907227, 0.5720647573471069, 0.6729124784469604], "cos_apical_negg": [0.1473718285560608, 0.14795580506324768, 0.143108069896698], "cos_Ac_negg": [0.5787884593009949, 0.5717729926109314, 0.672478437423706], "r_norm": [0.3370569348335266, 0.33832240104675293, 0.34798431396484375]}, {"step": 1800, "epoch": 3, "train_loss": 0.056912779808044434, "cos_r_negg": [0.14674267172813416, 0.1649877429008484, 0.17973743379116058], "cos_innovation_negg": [0.5927297472953796, 0.6014281511306763, 0.7244951128959656], "cos_apical_negg": [0.14674267172813416, 0.1649877429008484, 0.1797374188899994], "cos_Ac_negg": [0.5926502346992493, 0.6010866165161133, 0.7240047454833984], "r_norm": [0.4093218743801117, 0.413206547498703, 0.4214336574077606]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9572, "test_loss": 0.14480136337280272}, {"step": 1900, "epoch": 4, "train_loss": 0.06372798979282379, "cos_r_negg": [0.13798245787620544, 0.15645861625671387, 0.17202264070510864], "cos_innovation_negg": [0.5716080665588379, 0.5927119255065918, 0.7273725271224976], "cos_apical_negg": [0.13798245787620544, 0.15645861625671387, 0.17202265560626984], "cos_Ac_negg": [0.5702645182609558, 0.5924398899078369, 0.7267125248908997], "r_norm": [0.34442368149757385, 0.34690776467323303, 0.35579752922058105]}, {"step": 2000, "epoch": 4, "train_loss": 0.1230635792016983, "cos_r_negg": [0.09992668032646179, 0.1277817189693451, 0.17198091745376587], "cos_innovation_negg": [0.5658921003341675, 0.5797038078308105, 0.7345772981643677], "cos_apical_negg": [0.09992668032646179, 0.1277817189693451, 0.17198091745376587], "cos_Ac_negg": [0.5656217336654663, 0.5794889330863953, 0.7326558232307434], "r_norm": [0.34798136353492737, 0.34709346294403076, 0.3568878173828125], "ldr": {"dL_sdil": -0.008237451314926147, "dL_bp": -0.010793440043926239, "ratio": 0.7631905380862873}}, {"step": 2100, "epoch": 4, "train_loss": 0.0455523282289505, "cos_r_negg": [0.10512804985046387, 0.12953028082847595, 0.17792044579982758], "cos_innovation_negg": [0.5783050060272217, 0.5967017412185669, 0.6816978454589844], "cos_apical_negg": [0.10512804985046387, 0.12953028082847595, 0.17792044579982758], "cos_Ac_negg": [0.5776621103286743, 0.5964397192001343, 0.6802818179130554], "r_norm": [0.32047829031944275, 0.3206278085708618, 0.33012843132019043]}, {"step": 2200, "epoch": 4, "train_loss": 0.03517984598875046, "cos_r_negg": [0.12511026859283447, 0.10924628376960754, 0.1601351499557495], "cos_innovation_negg": [0.5803626775741577, 0.5878740549087524, 0.6889324188232422], "cos_apical_negg": [0.12511026859283447, 0.10924627631902695, 0.1601351499557495], "cos_Ac_negg": [0.5804789066314697, 0.5877931118011475, 0.687321662902832], "r_norm": [0.281892329454422, 0.28090548515319824, 0.2908461093902588]}, {"step": 2300, "epoch": 4, "train_loss": 0.24548226594924927, "cos_r_negg": [0.1039712056517601, 0.12470435351133347, 0.15579085052013397], "cos_innovation_negg": [0.5882450342178345, 0.594645082950592, 0.7426950335502625], "cos_apical_negg": [0.1039712056517601, 0.12470435351133347, 0.15579083561897278], "cos_Ac_negg": [0.5888385772705078, 0.5957909822463989, 0.7421364784240723], "r_norm": [0.28888705372810364, 0.2877044975757599, 0.29649096727371216]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9632, "test_loss": 0.13929505348205568}, {"step": 2400, "epoch": 5, "train_loss": 0.17687591910362244, "cos_r_negg": [0.09143434464931488, 0.09548715502023697, 0.14864963293075562], "cos_innovation_negg": [0.5699904561042786, 0.5822970867156982, 0.7467119693756104], "cos_apical_negg": [0.09143435209989548, 0.09548715502023697, 0.14864963293075562], "cos_Ac_negg": [0.5712879300117493, 0.583182692527771, 0.7456703782081604], "r_norm": [0.25851812958717346, 0.2587038278579712, 0.2650226950645447]}, {"step": 2500, "epoch": 5, "train_loss": 0.05235183238983154, "cos_r_negg": [0.104363352060318, 0.11514267325401306, 0.17557545006275177], "cos_innovation_negg": [0.5989686846733093, 0.6131246089935303, 0.7681947946548462], "cos_apical_negg": [0.104363352060318, 0.11514267325401306, 0.17557545006275177], "cos_Ac_negg": [0.5985739231109619, 0.6132676601409912, 0.7673127055168152], "r_norm": [0.26812252402305603, 0.26863086223602295, 0.2746120095252991], "ldr": {"dL_sdil": -0.005776338279247284, "dL_bp": -0.0076569244265556335, "ratio": 0.7543940566023967}}, {"step": 2600, "epoch": 5, "train_loss": 0.0871478021144867, "cos_r_negg": [0.10445903241634369, 0.10973095148801804, 0.16613927483558655], "cos_innovation_negg": [0.5762159824371338, 0.6001075506210327, 0.7818084955215454], "cos_apical_negg": [0.10445903241634369, 0.10973095148801804, 0.16613927483558655], "cos_Ac_negg": [0.5767149925231934, 0.6005728840827942, 0.7812463045120239], "r_norm": [0.2572261095046997, 0.25739797949790955, 0.2639195919036865]}, {"step": 2700, "epoch": 5, "train_loss": 0.03960831090807915, "cos_r_negg": [0.08011548966169357, 0.0950859785079956, 0.14954400062561035], "cos_innovation_negg": [0.599989116191864, 0.6124371886253357, 0.766748309135437], "cos_apical_negg": [0.08011548966169357, 0.0950859785079956, 0.14954400062561035], "cos_Ac_negg": [0.6005213856697083, 0.6123024225234985, 0.7662631273269653], "r_norm": [0.2452200949192047, 0.2448948621749878, 0.2524827718734741]}, {"step": 2800, "epoch": 5, "train_loss": 0.11025910079479218, "cos_r_negg": [0.08820969611406326, 0.08924748003482819, 0.13355904817581177], "cos_innovation_negg": [0.5641764402389526, 0.5957777500152588, 0.7485849857330322], "cos_apical_negg": [0.08820969611406326, 0.08924748748540878, 0.13355904817581177], "cos_Ac_negg": [0.5634011626243591, 0.5963249206542969, 0.7482997179031372], "r_norm": [0.23558063805103302, 0.23385974764823914, 0.24099965393543243]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9647, "test_loss": 0.1381507011413574}, {"step": 2900, "epoch": 6, "train_loss": 0.05658666789531708, "cos_r_negg": [0.08710053563117981, 0.10186932235956192, 0.13357140123844147], "cos_innovation_negg": [0.5684663653373718, 0.5904955863952637, 0.7487396597862244], "cos_apical_negg": [0.08710053563117981, 0.10186932235956192, 0.13357138633728027], "cos_Ac_negg": [0.569209098815918, 0.5921886563301086, 0.7497789859771729], "r_norm": [0.29279595613479614, 0.29504337906837463, 0.3026384711265564]}, {"step": 3000, "epoch": 6, "train_loss": 0.1389554738998413, "cos_r_negg": [0.08273432403802872, 0.10294032841920853, 0.153804212808609], "cos_innovation_negg": [0.581412136554718, 0.6046063899993896, 0.7675391435623169], "cos_apical_negg": [0.08273431658744812, 0.10294032096862793, 0.153804212808609], "cos_Ac_negg": [0.5810220837593079, 0.6044617891311646, 0.7668269276618958], "r_norm": [0.23665934801101685, 0.2363736927509308, 0.24261853098869324], "ldr": {"dL_sdil": -0.006766326725482941, "dL_bp": -0.010693289339542389, "ratio": 0.632763830719697}}, {"step": 3100, "epoch": 6, "train_loss": 0.045664429664611816, "cos_r_negg": [0.07281437516212463, 0.10329809784889221, 0.15000489354133606], "cos_innovation_negg": [0.5701096057891846, 0.6087615489959717, 0.7614849805831909], "cos_apical_negg": [0.07281437516212463, 0.10329809784889221, 0.15000489354133606], "cos_Ac_negg": [0.571969747543335, 0.6111091375350952, 0.7622292041778564], "r_norm": [0.21933972835540771, 0.21938401460647583, 0.2249061018228531]}, {"step": 3200, "epoch": 6, "train_loss": 0.061834823340177536, "cos_r_negg": [0.08501126617193222, 0.09319117665290833, 0.1612335443496704], "cos_innovation_negg": [0.5752924680709839, 0.6183765530586243, 0.7765172719955444], "cos_apical_negg": [0.08501126617193222, 0.09319118410348892, 0.1612335443496704], "cos_Ac_negg": [0.57663494348526, 0.6192549467086792, 0.7760336995124817], "r_norm": [0.2455293983221054, 0.24688546359539032, 0.2556905150413513]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9632, "test_loss": 0.13765224380493163}, {"step": 3300, "epoch": 7, "train_loss": 0.037868987768888474, "cos_r_negg": [0.06819568574428558, 0.09165281802415848, 0.13477952778339386], "cos_innovation_negg": [0.5775467753410339, 0.6172739267349243, 0.7733339667320251], "cos_apical_negg": [0.06819568574428558, 0.09165281802415848, 0.13477952778339386], "cos_Ac_negg": [0.5785447359085083, 0.6178483963012695, 0.772968053817749], "r_norm": [0.1841440498828888, 0.18279677629470825, 0.18698254227638245]}, {"step": 3400, "epoch": 7, "train_loss": 0.057012829929590225, "cos_r_negg": [0.04902275651693344, 0.0829361230134964, 0.1393163800239563], "cos_innovation_negg": [0.569525957107544, 0.611689031124115, 0.7531881332397461], "cos_apical_negg": [0.04902275651693344, 0.0829361230134964, 0.1393163800239563], "cos_Ac_negg": [0.5696339011192322, 0.6121697425842285, 0.7525510787963867], "r_norm": [0.2102440893650055, 0.21031144261360168, 0.21395905315876007]}, {"step": 3500, "epoch": 7, "train_loss": 0.012967953458428383, "cos_r_negg": [0.04261264204978943, 0.07681228965520859, 0.11562997847795486], "cos_innovation_negg": [0.5457855463027954, 0.5888035297393799, 0.7384692430496216], "cos_apical_negg": [0.04261264204978943, 0.07681229710578918, 0.11562997847795486], "cos_Ac_negg": [0.5459547638893127, 0.5891798138618469, 0.7380806803703308], "r_norm": [0.1930660754442215, 0.19278058409690857, 0.19642701745033264], "ldr": {"dL_sdil": -0.00444900244474411, "dL_bp": -0.005875870585441589, "ratio": 0.7571648115884694}}, {"step": 3600, "epoch": 7, "train_loss": 0.02658027783036232, "cos_r_negg": [0.06686761975288391, 0.09110414981842041, 0.14028650522232056], "cos_innovation_negg": [0.5533796548843384, 0.596343994140625, 0.7621746063232422], "cos_apical_negg": [0.06686761975288391, 0.09110414981842041, 0.14028650522232056], "cos_Ac_negg": [0.5548543334007263, 0.5974231958389282, 0.7601150274276733], "r_norm": [0.23936469852924347, 0.2389662116765976, 0.2441703826189041]}, {"step": 3700, "epoch": 7, "train_loss": 0.04484715685248375, "cos_r_negg": [0.07195615768432617, 0.09331950545310974, 0.1270626187324524], "cos_innovation_negg": [0.5737826824188232, 0.6007370948791504, 0.7920554876327515], "cos_apical_negg": [0.07195615768432617, 0.09331950545310974, 0.1270626187324524], "cos_Ac_negg": [0.5744510889053345, 0.600490927696228, 0.7909368276596069], "r_norm": [0.21639801561832428, 0.216265469789505, 0.22190241515636444]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.964, "test_loss": 0.13408593940734864}, {"step": 3800, "epoch": 8, "train_loss": 0.01185213029384613, "cos_r_negg": [0.06454490125179291, 0.07939378917217255, 0.11251212656497955], "cos_innovation_negg": [0.5821287631988525, 0.611849308013916, 0.8006997108459473], "cos_apical_negg": [0.06454490125179291, 0.07939378917217255, 0.11251212656497955], "cos_Ac_negg": [0.5833477973937988, 0.6126703023910522, 0.8004059791564941], "r_norm": [0.20735469460487366, 0.20676398277282715, 0.21254190802574158]}, {"step": 3900, "epoch": 8, "train_loss": 0.05419771373271942, "cos_r_negg": [0.06062908470630646, 0.0694422721862793, 0.14149636030197144], "cos_innovation_negg": [0.589740514755249, 0.609463095664978, 0.7986055016517639], "cos_apical_negg": [0.060629088431596756, 0.0694422721862793, 0.14149636030197144], "cos_Ac_negg": [0.5911917686462402, 0.611224889755249, 0.7968090176582336], "r_norm": [0.23106130957603455, 0.22936022281646729, 0.23741474747657776]}, {"step": 4000, "epoch": 8, "train_loss": 0.046872012317180634, "cos_r_negg": [0.05422832816839218, 0.06284771859645844, 0.1561664193868637], "cos_innovation_negg": [0.5773699283599854, 0.6160436868667603, 0.7888002395629883], "cos_apical_negg": [0.05422832816839218, 0.06284771859645844, 0.1561664193868637], "cos_Ac_negg": [0.579522430896759, 0.617862343788147, 0.7886414527893066], "r_norm": [0.19908982515335083, 0.19718077778816223, 0.20243209600448608], "ldr": {"dL_sdil": -0.004139915108680725, "dL_bp": -0.007579512894153595, "ratio": 0.5461980428681664}}, {"step": 4100, "epoch": 8, "train_loss": 0.03398824483156204, "cos_r_negg": [0.06557351350784302, 0.07014419138431549, 0.12502682209014893], "cos_innovation_negg": [0.5768357515335083, 0.6040399074554443, 0.773569643497467], "cos_apical_negg": [0.06557351350784302, 0.0701441839337349, 0.12502682209014893], "cos_Ac_negg": [0.5798871517181396, 0.6070576906204224, 0.7739890813827515], "r_norm": [0.21416997909545898, 0.21151283383369446, 0.21724577248096466]}, {"step": 4200, "epoch": 8, "train_loss": 0.07612821459770203, "cos_r_negg": [0.06579138338565826, 0.08105255663394928, 0.15147385001182556], "cos_innovation_negg": [0.5834667682647705, 0.6256547570228577, 0.7978948950767517], "cos_apical_negg": [0.06579138338565826, 0.08105255663394928, 0.15147385001182556], "cos_Ac_negg": [0.5845136046409607, 0.6251918077468872, 0.7970051765441895], "r_norm": [0.20306122303009033, 0.20107214152812958, 0.20588289201259613]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9662, "test_loss": 0.14420917987823487}, {"step": 4300, "epoch": 9, "train_loss": 0.08293405175209045, "cos_r_negg": [0.0714031234383583, 0.0785452127456665, 0.15151631832122803], "cos_innovation_negg": [0.5730950832366943, 0.6166566014289856, 0.7786111235618591], "cos_apical_negg": [0.0714031234383583, 0.07854520529508591, 0.15151631832122803], "cos_Ac_negg": [0.5783324837684631, 0.6188297271728516, 0.7781251072883606], "r_norm": [0.24066542088985443, 0.23902548849582672, 0.24453666806221008]}, {"step": 4400, "epoch": 9, "train_loss": 0.01634911447763443, "cos_r_negg": [0.04901481419801712, 0.06164327636361122, 0.1508701741695404], "cos_innovation_negg": [0.5692999362945557, 0.6197838187217712, 0.7927394509315491], "cos_apical_negg": [0.04901481419801712, 0.06164327263832092, 0.1508701741695404], "cos_Ac_negg": [0.57411789894104, 0.6215963959693909, 0.7921978235244751], "r_norm": [0.19413670897483826, 0.19463616609573364, 0.19877862930297852]}, {"step": 4500, "epoch": 9, "train_loss": 0.08244402706623077, "cos_r_negg": [0.07131268084049225, 0.06934712082147598, 0.170973539352417], "cos_innovation_negg": [0.5771777629852295, 0.6264559626579285, 0.7843647003173828], "cos_apical_negg": [0.07131268084049225, 0.06934712082147598, 0.170973539352417], "cos_Ac_negg": [0.5818760395050049, 0.6306333541870117, 0.7838480472564697], "r_norm": [0.21711620688438416, 0.21687400341033936, 0.2219024896621704], "ldr": {"dL_sdil": -0.009840846061706543, "dL_bp": -0.015845887362957, "ratio": 0.6210347099091169}}, {"step": 4600, "epoch": 9, "train_loss": 0.026744356378912926, "cos_r_negg": [0.0618271604180336, 0.06315726041793823, 0.16725903749465942], "cos_innovation_negg": [0.5672653317451477, 0.6101313233375549, 0.7892102003097534], "cos_apical_negg": [0.0618271566927433, 0.06315726041793823, 0.16725903749465942], "cos_Ac_negg": [0.5734517574310303, 0.612655520439148, 0.7888774275779724], "r_norm": [0.22485798597335815, 0.2251223921775818, 0.2306308001279831]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9667, "test_loss": 0.13903922443389893}, {"step": 4700, "epoch": 10, "train_loss": 0.007480492815375328, "cos_r_negg": [0.057246074080467224, 0.07192473113536835, 0.12905441224575043], "cos_innovation_negg": [0.5789980888366699, 0.6045011281967163, 0.7860550880432129], "cos_apical_negg": [0.057246070355176926, 0.07192473113536835, 0.12905442714691162], "cos_Ac_negg": [0.5840855836868286, 0.6071586608886719, 0.7865488529205322], "r_norm": [0.19419127702713013, 0.19209963083267212, 0.19691956043243408]}, {"step": 4800, "epoch": 10, "train_loss": 0.05859462171792984, "cos_r_negg": [0.048658277839422226, 0.049405887722969055, 0.1246444433927536], "cos_innovation_negg": [0.580818235874176, 0.6091240048408508, 0.8054152131080627], "cos_apical_negg": [0.04865827411413193, 0.049405887722969055, 0.1246444433927536], "cos_Ac_negg": [0.5851349830627441, 0.6118993163108826, 0.8056197762489319], "r_norm": [0.16325904428958893, 0.16174930334091187, 0.1662379950284958]}, {"step": 4900, "epoch": 10, "train_loss": 0.01692567951977253, "cos_r_negg": [0.041120436042547226, 0.052641406655311584, 0.13419954478740692], "cos_innovation_negg": [0.5894560813903809, 0.6101266145706177, 0.8228746652603149], "cos_apical_negg": [0.041120436042547226, 0.052641406655311584, 0.13419952988624573], "cos_Ac_negg": [0.5942845344543457, 0.6120989322662354, 0.8230128288269043], "r_norm": [0.21571972966194153, 0.21420744061470032, 0.21982790529727936]}, {"step": 5000, "epoch": 10, "train_loss": 0.002963550854474306, "cos_r_negg": [0.02652878873050213, 0.054692912846803665, 0.14462882280349731], "cos_innovation_negg": [0.5711955428123474, 0.6041380167007446, 0.7934826612472534], "cos_apical_negg": [0.02652878686785698, 0.054692916572093964, 0.14462882280349731], "cos_Ac_negg": [0.5788955688476562, 0.6082943677902222, 0.7967605590820312], "r_norm": [0.19515369832515717, 0.1932317167520523, 0.1971580982208252], "ldr": {"dL_sdil": -0.006302967667579651, "dL_bp": -0.013330258429050446, "ratio": 0.47283161846612676}}, {"step": 5100, "epoch": 10, "train_loss": 0.06172416731715202, "cos_r_negg": [0.049603741616010666, 0.0570407509803772, 0.10463455319404602], "cos_innovation_negg": [0.5679624676704407, 0.6072711944580078, 0.8077365756034851], "cos_apical_negg": [0.04960373416543007, 0.057040754705667496, 0.10463454574346542], "cos_Ac_negg": [0.5724920034408569, 0.6101170778274536, 0.8092787861824036], "r_norm": [0.20149093866348267, 0.2006198763847351, 0.20440885424613953]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9663, "test_loss": 0.13990628299713134}, {"step": 5200, "epoch": 11, "train_loss": 0.005635946057736874, "cos_r_negg": [0.04804793745279312, 0.07594914734363556, 0.15396186709403992], "cos_innovation_negg": [0.5863626003265381, 0.6298747062683105, 0.8127346038818359], "cos_apical_negg": [0.04804793745279312, 0.07594914734363556, 0.15396186709403992], "cos_Ac_negg": [0.5908218622207642, 0.6329330205917358, 0.8132315874099731], "r_norm": [0.19912630319595337, 0.19829660654067993, 0.20222434401512146]}, {"step": 5300, "epoch": 11, "train_loss": 0.04544731229543686, "cos_r_negg": [0.045524321496486664, 0.05399655923247337, 0.13895809650421143], "cos_innovation_negg": [0.592345118522644, 0.6250184774398804, 0.8202204704284668], "cos_apical_negg": [0.045524321496486664, 0.05399656295776367, 0.13895811140537262], "cos_Ac_negg": [0.5951798558235168, 0.6259604692459106, 0.817613959312439], "r_norm": [0.17030401527881622, 0.16832587122917175, 0.17232629656791687]}, {"step": 5400, "epoch": 11, "train_loss": 0.032391257584095, "cos_r_negg": [0.033009544014930725, 0.06161931902170181, 0.11818663775920868], "cos_innovation_negg": [0.585328996181488, 0.6277340650558472, 0.8154817819595337], "cos_apical_negg": [0.033009547740221024, 0.06161931902170181, 0.11818663775920868], "cos_Ac_negg": [0.5880820155143738, 0.6308137774467468, 0.8149290084838867], "r_norm": [0.16117753088474274, 0.1590377688407898, 0.16213363409042358]}, {"step": 5500, "epoch": 11, "train_loss": 0.017001952975988388, "cos_r_negg": [0.057188842445611954, 0.06892257928848267, 0.14046448469161987], "cos_innovation_negg": [0.5733467936515808, 0.6209812760353088, 0.8231766223907471], "cos_apical_negg": [0.057188838720321655, 0.06892257928848267, 0.14046448469161987], "cos_Ac_negg": [0.5821273326873779, 0.6259586811065674, 0.8244051337242126], "r_norm": [0.20744086802005768, 0.20738153159618378, 0.21173736453056335], "ldr": {"dL_sdil": -0.00945746898651123, "dL_bp": -0.02136269211769104, "ratio": 0.4427096048760276}}, {"step": 5600, "epoch": 11, "train_loss": 0.006740005221217871, "cos_r_negg": [0.05281400680541992, 0.05836670100688934, 0.14640334248542786], "cos_innovation_negg": [0.5676909685134888, 0.6017760038375854, 0.7918360233306885], "cos_apical_negg": [0.05281400308012962, 0.05836670100688934, 0.14640334248542786], "cos_Ac_negg": [0.5811183452606201, 0.613360583782196, 0.7984145879745483], "r_norm": [0.1922435462474823, 0.1913505494594574, 0.194338858127594]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9681, "test_loss": 0.13979484329223632}, {"step": 5700, "epoch": 12, "train_loss": 0.00581244844943285, "cos_r_negg": [0.04240035265684128, 0.06806708872318268, 0.15737377107143402], "cos_innovation_negg": [0.5983659029006958, 0.6389483213424683, 0.8126609325408936], "cos_apical_negg": [0.04240035638213158, 0.06806708872318268, 0.15737377107143402], "cos_Ac_negg": [0.6039735674858093, 0.6432410478591919, 0.8180676698684692], "r_norm": [0.18136031925678253, 0.17909833788871765, 0.18332219123840332]}, {"step": 5800, "epoch": 12, "train_loss": 0.00885578989982605, "cos_r_negg": [0.0430121123790741, 0.06361928582191467, 0.16144409775733948], "cos_innovation_negg": [0.5721216201782227, 0.6162316799163818, 0.8096988201141357], "cos_apical_negg": [0.0430121123790741, 0.06361928582191467, 0.16144409775733948], "cos_Ac_negg": [0.5854752659797668, 0.6243906617164612, 0.8141613006591797], "r_norm": [0.17212796211242676, 0.17050936818122864, 0.17365111410617828]}, {"step": 5900, "epoch": 12, "train_loss": 0.009952781721949577, "cos_r_negg": [0.047281891107559204, 0.07056909799575806, 0.12812644243240356], "cos_innovation_negg": [0.5634245276451111, 0.6183832287788391, 0.8073009252548218], "cos_apical_negg": [0.047281887382268906, 0.07056909799575806, 0.12812642753124237], "cos_Ac_negg": [0.5754376649856567, 0.626757025718689, 0.8092242479324341], "r_norm": [0.1940169483423233, 0.19259941577911377, 0.19687262177467346]}, {"step": 6000, "epoch": 12, "train_loss": 0.01151820458471775, "cos_r_negg": [0.051623858511447906, 0.06779216229915619, 0.11005212366580963], "cos_innovation_negg": [0.5707551836967468, 0.6161702871322632, 0.8066044449806213], "cos_apical_negg": [0.051623858511447906, 0.06779216229915619, 0.11005212366580963], "cos_Ac_negg": [0.5827686786651611, 0.6238081455230713, 0.8084499835968018], "r_norm": [0.186072438955307, 0.18489909172058105, 0.1888715922832489], "ldr": {"dL_sdil": -0.006349913775920868, "dL_bp": -0.013403870165348053, "ratio": 0.4737373383649141}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9675, "test_loss": 0.1411269510269165}, {"step": 6100, "epoch": 13, "train_loss": 0.023034799844026566, "cos_r_negg": [0.02535937912762165, 0.05402741953730583, 0.11632424592971802], "cos_innovation_negg": [0.577201247215271, 0.6136166453361511, 0.798707902431488], "cos_apical_negg": [0.0253593772649765, 0.05402742326259613, 0.11632424592971802], "cos_Ac_negg": [0.5856450796127319, 0.619591474533081, 0.797835111618042], "r_norm": [0.16720417141914368, 0.1653231382369995, 0.16929173469543457]}, {"step": 6200, "epoch": 13, "train_loss": 0.006544003263115883, "cos_r_negg": [0.01941419020295143, 0.052159279584884644, 0.10601349174976349], "cos_innovation_negg": [0.5653433799743652, 0.617152214050293, 0.7921702265739441], "cos_apical_negg": [0.01941419020295143, 0.052159279584884644, 0.10601348429918289], "cos_Ac_negg": [0.5742806196212769, 0.6222989559173584, 0.7963885068893433], "r_norm": [0.17755120992660522, 0.1749841570854187, 0.1788170337677002]}, {"step": 6300, "epoch": 13, "train_loss": 0.004289270378649235, "cos_r_negg": [0.0340130552649498, 0.05541940778493881, 0.08644931763410568], "cos_innovation_negg": [0.5821785926818848, 0.6249187588691711, 0.8116950988769531], "cos_apical_negg": [0.0340130552649498, 0.05541940778493881, 0.08644931763410568], "cos_Ac_negg": [0.590609073638916, 0.6309099197387695, 0.8122329711914062], "r_norm": [0.18385039269924164, 0.1820000410079956, 0.18596194684505463]}, {"step": 6400, "epoch": 13, "train_loss": 0.010423911735415459, "cos_r_negg": [0.03649044781923294, 0.060498591512441635, 0.1368914395570755], "cos_innovation_negg": [0.5671042203903198, 0.6166531443595886, 0.810198187828064], "cos_apical_negg": [0.03649044781923294, 0.060498595237731934, 0.1368914395570755], "cos_Ac_negg": [0.5799421072006226, 0.6260737776756287, 0.8127725124359131], "r_norm": [0.18087336421012878, 0.17894110083580017, 0.1818162500858307]}, {"step": 6500, "epoch": 13, "train_loss": 0.004160104785114527, "cos_r_negg": [0.0405738428235054, 0.04769139736890793, 0.09347885847091675], "cos_innovation_negg": [0.5765970349311829, 0.6279065608978271, 0.8124428987503052], "cos_apical_negg": [0.0405738428235054, 0.04769139736890793, 0.09347885847091675], "cos_Ac_negg": [0.5845774412155151, 0.6358634233474731, 0.8168673515319824], "r_norm": [0.1860228180885315, 0.18419817090034485, 0.18813836574554443], "ldr": {"dL_sdil": -0.010338686406612396, "dL_bp": -0.022747255861759186, "ratio": 0.454502576901724}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9682, "test_loss": 0.1463049343109131}, {"step": 6600, "epoch": 14, "train_loss": 0.004148650448769331, "cos_r_negg": [0.03188185393810272, 0.06319305300712585, 0.13173982501029968], "cos_innovation_negg": [0.5802109241485596, 0.6224278211593628, 0.8078254461288452], "cos_apical_negg": [0.03188186138868332, 0.06319305300712585, 0.13173982501029968], "cos_Ac_negg": [0.5904284119606018, 0.6292195916175842, 0.8099945783615112], "r_norm": [0.1784917116165161, 0.17618262767791748, 0.18012207746505737]}, {"step": 6700, "epoch": 14, "train_loss": 0.026638057082891464, "cos_r_negg": [0.049549493938684464, 0.07205605506896973, 0.12926535308361053], "cos_innovation_negg": [0.5660225749015808, 0.6206774711608887, 0.8216431140899658], "cos_apical_negg": [0.04954949766397476, 0.07205605506896973, 0.12926535308361053], "cos_Ac_negg": [0.578456699848175, 0.6301831007003784, 0.8272661566734314], "r_norm": [0.20544731616973877, 0.20489473640918732, 0.2076413929462433]}, {"step": 6800, "epoch": 14, "train_loss": 0.009142404422163963, "cos_r_negg": [0.0335632786154747, 0.0695372074842453, 0.09248542040586472], "cos_innovation_negg": [0.5792226791381836, 0.6255484223365784, 0.8168814182281494], "cos_apical_negg": [0.0335632786154747, 0.0695372074842453, 0.09248542040586472], "cos_Ac_negg": [0.5914014577865601, 0.632810115814209, 0.8217812180519104], "r_norm": [0.16425728797912598, 0.1625906080007553, 0.16580429673194885]}, {"step": 6900, "epoch": 14, "train_loss": 0.017450060695409775, "cos_r_negg": [0.024999558925628662, 0.05355795472860336, 0.108038529753685], "cos_innovation_negg": [0.5662834644317627, 0.6149284839630127, 0.8097664713859558], "cos_apical_negg": [0.02499956078827381, 0.05355795472860336, 0.108038529753685], "cos_Ac_negg": [0.5823071002960205, 0.6273297071456909, 0.8186169862747192], "r_norm": [0.16578197479248047, 0.16436287760734558, 0.16795970499515533]}, {"step": 7000, "epoch": 14, "train_loss": 0.004938879981637001, "cos_r_negg": [0.019482914358377457, 0.04799122363328934, 0.12605518102645874], "cos_innovation_negg": [0.5600874423980713, 0.6135075688362122, 0.8163824081420898], "cos_apical_negg": [0.019482912495732307, 0.04799122363328934, 0.12605518102645874], "cos_Ac_negg": [0.5771578550338745, 0.6272215247154236, 0.8234264850616455], "r_norm": [0.17362800240516663, 0.1731886863708496, 0.17663457989692688], "ldr": {"dL_sdil": -0.007719583809375763, "dL_bp": -0.01837293803691864, "ratio": 0.42016055319318046}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9688, "test_loss": 0.14673759593963623}], "final": {"test_acc": 0.9688, "test_loss": 0.14673759593963623, "wall_s": 24.029053211212158, "cos_r_negg": [0.01788480021059513, 0.04366231709718704, 0.11157252639532089], "cos_innovation_negg": [0.5543270111083984, 0.6096324920654297, 0.8096452355384827], "cos_apical_negg": [0.01788480207324028, 0.04366231709718704, 0.11157253384590149], "cos_Ac_negg": [0.5765706300735474, 0.6261533498764038, 0.8202852606773376]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_raw_s0.json b/results/nuis_ctrl_v1_rho0p05_raw_s0.json new file mode 100644 index 0000000..a88a698 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_raw_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_raw_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.018889345228672028, -0.02601044997572899, 0.032917968928813934], "cos_innovation_negg": [-0.01886402815580368, -0.02586054801940918, 0.03356533125042915], "cos_apical_negg": [-0.018889345228672028, -0.02601044997572899, 0.032917968928813934], "cos_Ac_negg": [-0.018863998353481293, -0.02586054801940918, 0.03356531634926796], "r_norm": [4.982118606567383, 4.8570404052734375, 4.847076892852783], "ldr": {"dL_sdil": -0.008685588836669922, "dL_bp": -0.23511672019958496, "ratio": 0.03694160427764105}}, {"step": 100, "epoch": 0, "train_loss": 0.2316625416278839, "cos_r_negg": [0.23470458388328552, 0.23169171810150146, 0.19987548887729645], "cos_innovation_negg": [0.45221105217933655, 0.43953999876976013, 0.4016229808330536], "cos_apical_negg": [0.23470458388328552, 0.23169171810150146, 0.19987548887729645], "cos_Ac_negg": [0.45217204093933105, 0.43955740332603455, 0.40168216824531555], "r_norm": [1.4844417572021484, 1.4598970413208008, 1.4545302391052246]}, {"step": 200, "epoch": 0, "train_loss": 0.3199096918106079, "cos_r_negg": [0.19267380237579346, 0.24690712988376617, 0.24164065718650818], "cos_innovation_negg": [0.4661179780960083, 0.46744492650032043, 0.41031360626220703], "cos_apical_negg": [0.19267380237579346, 0.24690712988376617, 0.24164065718650818], "cos_Ac_negg": [0.4661034941673279, 0.4674678444862366, 0.41032806038856506], "r_norm": [1.3621758222579956, 1.3459157943725586, 1.341383934020996]}, {"step": 300, "epoch": 0, "train_loss": 0.22720038890838623, "cos_r_negg": [0.20497924089431763, 0.2550729513168335, 0.2500893175601959], "cos_innovation_negg": [0.4957374930381775, 0.5101429224014282, 0.4892195165157318], "cos_apical_negg": [0.20497924089431763, 0.2550729513168335, 0.2500893175601959], "cos_Ac_negg": [0.4957354664802551, 0.5101265907287598, 0.48918506503105164], "r_norm": [1.2987923622131348, 1.2880769968032837, 1.2851369380950928]}, {"step": 400, "epoch": 0, "train_loss": 0.16089418530464172, "cos_r_negg": [0.19641119241714478, 0.22100770473480225, 0.22859039902687073], "cos_innovation_negg": [0.532158374786377, 0.5136614441871643, 0.4884665608406067], "cos_apical_negg": [0.19641119241714478, 0.22100770473480225, 0.22859039902687073], "cos_Ac_negg": [0.5320866703987122, 0.5137494802474976, 0.488487184047699], "r_norm": [1.2419109344482422, 1.2315348386764526, 1.231013536453247]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9414, "test_loss": 0.21403483505249024}, {"step": 500, "epoch": 1, "train_loss": 0.22588635981082916, "cos_r_negg": [0.14134198427200317, 0.2137194722890854, 0.2176074981689453], "cos_innovation_negg": [0.4921151399612427, 0.48354625701904297, 0.46166861057281494], "cos_apical_negg": [0.14134198427200317, 0.2137194722890854, 0.2176074981689453], "cos_Ac_negg": [0.49202507734298706, 0.4835682809352875, 0.46145182847976685], "r_norm": [1.2109417915344238, 1.200516939163208, 1.1987390518188477], "ldr": {"dL_sdil": -0.008084222674369812, "dL_bp": -0.006889566779136658, "ratio": 1.1734007280183238}}, {"step": 600, "epoch": 1, "train_loss": 0.1248762458562851, "cos_r_negg": [0.16898371279239655, 0.2130759358406067, 0.29900920391082764], "cos_innovation_negg": [0.47226765751838684, 0.47323399782180786, 0.5102213621139526], "cos_apical_negg": [0.16898371279239655, 0.2130759358406067, 0.29900920391082764], "cos_Ac_negg": [0.47219038009643555, 0.47315865755081177, 0.5101075768470764], "r_norm": [1.2194931507110596, 1.2160041332244873, 1.207112193107605]}, {"step": 700, "epoch": 1, "train_loss": 0.19223998486995697, "cos_r_negg": [0.14750772714614868, 0.20457768440246582, 0.2899758815765381], "cos_innovation_negg": [0.4767991900444031, 0.46232789754867554, 0.49343907833099365], "cos_apical_negg": [0.14750772714614868, 0.20457768440246582, 0.2899758815765381], "cos_Ac_negg": [0.47682324051856995, 0.4623897969722748, 0.4933580756187439], "r_norm": [1.2216745615005493, 1.2105355262756348, 1.20802903175354]}, {"step": 800, "epoch": 1, "train_loss": 0.15692606568336487, "cos_r_negg": [0.13524428009986877, 0.21434614062309265, 0.2651273012161255], "cos_innovation_negg": [0.4759499728679657, 0.4723067879676819, 0.476607084274292], "cos_apical_negg": [0.13524428009986877, 0.21434614062309265, 0.2651273012161255], "cos_Ac_negg": [0.47590887546539307, 0.47228991985321045, 0.47643569111824036], "r_norm": [1.1303777694702148, 1.1217707395553589, 1.1227798461914062]}, {"step": 900, "epoch": 1, "train_loss": 0.17302459478378296, "cos_r_negg": [0.1526738405227661, 0.2253347635269165, 0.3220912218093872], "cos_innovation_negg": [0.5346554517745972, 0.5626517534255981, 0.566016674041748], "cos_apical_negg": [0.1526738405227661, 0.2253347635269165, 0.3220912218093872], "cos_Ac_negg": [0.534640908241272, 0.5626353025436401, 0.5658434629440308], "r_norm": [1.147991418838501, 1.1383893489837646, 1.1372339725494385]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9454, "test_loss": 0.20151613082885741}, {"step": 1000, "epoch": 2, "train_loss": 0.05331810563802719, "cos_r_negg": [0.11788767576217651, 0.17947617173194885, 0.35381388664245605], "cos_innovation_negg": [0.4849860966205597, 0.5148710012435913, 0.5581153035163879], "cos_apical_negg": [0.11788767576217651, 0.17947617173194885, 0.35381388664245605], "cos_Ac_negg": [0.48491957783699036, 0.5147826671600342, 0.5579615831375122], "r_norm": [1.1214679479599, 1.1133793592453003, 1.1112723350524902], "ldr": {"dL_sdil": -0.007101535797119141, "dL_bp": -0.0070992112159729, "ratio": 1.0003274421728727}}, {"step": 1100, "epoch": 2, "train_loss": 0.1732659488916397, "cos_r_negg": [0.11807993799448013, 0.17467465996742249, 0.34313422441482544], "cos_innovation_negg": [0.47264716029167175, 0.48618900775909424, 0.5216293334960938], "cos_apical_negg": [0.11807993799448013, 0.17467465996742249, 0.34313422441482544], "cos_Ac_negg": [0.4726192355155945, 0.48613595962524414, 0.5215154886245728], "r_norm": [1.1383291482925415, 1.1279253959655762, 1.1292471885681152]}, {"step": 1200, "epoch": 2, "train_loss": 0.21508166193962097, "cos_r_negg": [0.11980476975440979, 0.157465398311615, 0.2798289656639099], "cos_innovation_negg": [0.4708786606788635, 0.4826098084449768, 0.5160412788391113], "cos_apical_negg": [0.11980476975440979, 0.157465398311615, 0.2798289656639099], "cos_Ac_negg": [0.470876544713974, 0.4825723469257355, 0.5159133076667786], "r_norm": [1.1140010356903076, 1.1040006875991821, 1.1062731742858887]}, {"step": 1300, "epoch": 2, "train_loss": 0.14503014087677002, "cos_r_negg": [0.15452975034713745, 0.17584452033042908, 0.3179256021976471], "cos_innovation_negg": [0.4997187852859497, 0.5223895311355591, 0.47794729471206665], "cos_apical_negg": [0.15452975034713745, 0.17584452033042908, 0.3179256021976471], "cos_Ac_negg": [0.4996158182621002, 0.5223785042762756, 0.47774845361709595], "r_norm": [1.112052321434021, 1.1025787591934204, 1.1016534566879272]}, {"step": 1400, "epoch": 2, "train_loss": 0.17568084597587585, "cos_r_negg": [0.09762503951787949, 0.1793743073940277, 0.3626384437084198], "cos_innovation_negg": [0.5365784764289856, 0.561907172203064, 0.5673563480377197], "cos_apical_negg": [0.09762503951787949, 0.1793743073940277, 0.3626384437084198], "cos_Ac_negg": [0.5362609624862671, 0.5614597797393799, 0.566525936126709], "r_norm": [1.1025522947311401, 1.095109224319458, 1.0913634300231934]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9572, "test_loss": 0.16043873100280762}, {"step": 1500, "epoch": 3, "train_loss": 0.1210009902715683, "cos_r_negg": [0.11008252948522568, 0.22029989957809448, 0.3482934832572937], "cos_innovation_negg": [0.48705393075942993, 0.5561088919639587, 0.5838615894317627], "cos_apical_negg": [0.11008252948522568, 0.22029989957809448, 0.3482934832572937], "cos_Ac_negg": [0.4869464337825775, 0.5559383630752563, 0.5836822986602783], "r_norm": [1.069291353225708, 1.0639028549194336, 1.0641794204711914], "ldr": {"dL_sdil": -0.0044784098863601685, "dL_bp": -0.004492044448852539, "ratio": 0.9969647311713815}}, {"step": 1600, "epoch": 3, "train_loss": 0.09388969838619232, "cos_r_negg": [0.08775987476110458, 0.15973791480064392, 0.25329574942588806], "cos_innovation_negg": [0.4820828139781952, 0.5090473890304565, 0.53265780210495], "cos_apical_negg": [0.08775987476110458, 0.15973791480064392, 0.25329574942588806], "cos_Ac_negg": [0.4820255935192108, 0.5089107751846313, 0.5325554609298706], "r_norm": [1.07172429561615, 1.0621823072433472, 1.062644124031067]}, {"step": 1700, "epoch": 3, "train_loss": 0.07022413611412048, "cos_r_negg": [0.10848211497068405, 0.17679093778133392, 0.29205521941185], "cos_innovation_negg": [0.4823445677757263, 0.5424801111221313, 0.5775178074836731], "cos_apical_negg": [0.10848211497068405, 0.17679093778133392, 0.29205521941185], "cos_Ac_negg": [0.4822928309440613, 0.542522668838501, 0.5774283409118652], "r_norm": [1.089035153388977, 1.0836055278778076, 1.0805248022079468]}, {"step": 1800, "epoch": 3, "train_loss": 0.038585830479860306, "cos_r_negg": [0.10026787966489792, 0.1584472954273224, 0.2623743414878845], "cos_innovation_negg": [0.4616909325122833, 0.49445784091949463, 0.5743393898010254], "cos_apical_negg": [0.10026787966489792, 0.1584472954273224, 0.2623743414878845], "cos_Ac_negg": [0.4615846872329712, 0.4943014979362488, 0.5740854740142822], "r_norm": [1.0575604438781738, 1.051081657409668, 1.0498106479644775]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9567, "test_loss": 0.1669241989135742}, {"step": 1900, "epoch": 4, "train_loss": 0.05220799520611763, "cos_r_negg": [0.12107962369918823, 0.19493287801742554, 0.3175318241119385], "cos_innovation_negg": [0.49412333965301514, 0.5541813373565674, 0.576842725276947], "cos_apical_negg": [0.12107962369918823, 0.19493287801742554, 0.3175318241119385], "cos_Ac_negg": [0.49395525455474854, 0.5539348125457764, 0.576566219329834], "r_norm": [1.04732346534729, 1.0417029857635498, 1.0409610271453857]}, {"step": 2000, "epoch": 4, "train_loss": 0.1620032787322998, "cos_r_negg": [0.09274173527956009, 0.18109172582626343, 0.2818826138973236], "cos_innovation_negg": [0.4649173319339752, 0.48723775148391724, 0.5584830045700073], "cos_apical_negg": [0.09274173527956009, 0.18109172582626343, 0.2818826138973236], "cos_Ac_negg": [0.46477028727531433, 0.4870375394821167, 0.5584304928779602], "r_norm": [1.039423942565918, 1.035604476928711, 1.0339125394821167], "ldr": {"dL_sdil": -0.005337387323379517, "dL_bp": -0.007765054702758789, "ratio": 0.6873599128005158}}, {"step": 2100, "epoch": 4, "train_loss": 0.05869222432374954, "cos_r_negg": [0.13352806866168976, 0.21133017539978027, 0.2777782082557678], "cos_innovation_negg": [0.44825026392936707, 0.48020094633102417, 0.5685089230537415], "cos_apical_negg": [0.13352806866168976, 0.21133017539978027, 0.2777782082557678], "cos_Ac_negg": [0.44824129343032837, 0.4801056683063507, 0.568452000617981], "r_norm": [1.0415667295455933, 1.0398322343826294, 1.037940502166748]}, {"step": 2200, "epoch": 4, "train_loss": 0.05471344292163849, "cos_r_negg": [0.08597137033939362, 0.17008939385414124, 0.25793978571891785], "cos_innovation_negg": [0.4834924042224884, 0.5145854949951172, 0.6063051819801331], "cos_apical_negg": [0.08597137033939362, 0.17008939385414124, 0.25793978571891785], "cos_Ac_negg": [0.4834393262863159, 0.514431357383728, 0.6061898469924927], "r_norm": [1.0674641132354736, 1.0618911981582642, 1.0563780069351196]}, {"step": 2300, "epoch": 4, "train_loss": 0.29510194063186646, "cos_r_negg": [0.11807355284690857, 0.20011469721794128, 0.29283344745635986], "cos_innovation_negg": [0.4751397371292114, 0.5576819181442261, 0.6443016529083252], "cos_apical_negg": [0.11807355284690857, 0.20011469721794128, 0.29283344745635986], "cos_Ac_negg": [0.47499263286590576, 0.5572212934494019, 0.6441560387611389], "r_norm": [1.061295747756958, 1.0528438091278076, 1.0521645545959473]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9605, "test_loss": 0.15810680198669433}, {"step": 2400, "epoch": 5, "train_loss": 0.11830407381057739, "cos_r_negg": [0.06817568093538284, 0.1392599493265152, 0.30653971433639526], "cos_innovation_negg": [0.43076619505882263, 0.5064486861228943, 0.6709274053573608], "cos_apical_negg": [0.06817568093538284, 0.1392599493265152, 0.30653971433639526], "cos_Ac_negg": [0.4307430684566498, 0.5063877105712891, 0.6708958148956299], "r_norm": [1.0599803924560547, 1.0531288385391235, 1.0495966672897339]}, {"step": 2500, "epoch": 5, "train_loss": 0.03706478327512741, "cos_r_negg": [0.09750305861234665, 0.18643105030059814, 0.2588551640510559], "cos_innovation_negg": [0.45465922355651855, 0.5231276750564575, 0.6625532507896423], "cos_apical_negg": [0.09750305861234665, 0.18643105030059814, 0.2588551640510559], "cos_Ac_negg": [0.4545869827270508, 0.5228502750396729, 0.6626551747322083], "r_norm": [1.0685884952545166, 1.062732458114624, 1.0581591129302979], "ldr": {"dL_sdil": -0.008816704154014587, "dL_bp": -0.010646924376487732, "ratio": 0.8280986923777787}}, {"step": 2600, "epoch": 5, "train_loss": 0.0740327388048172, "cos_r_negg": [0.09968943893909454, 0.18072400987148285, 0.26391273736953735], "cos_innovation_negg": [0.4368077516555786, 0.5317527055740356, 0.7128058671951294], "cos_apical_negg": [0.09968943893909454, 0.18072400987148285, 0.26391273736953735], "cos_Ac_negg": [0.4366735517978668, 0.5316020250320435, 0.7130361795425415], "r_norm": [1.0688101053237915, 1.0605977773666382, 1.0566633939743042]}, {"step": 2700, "epoch": 5, "train_loss": 0.0685981884598732, "cos_r_negg": [0.11481891572475433, 0.20972947776317596, 0.2915850877761841], "cos_innovation_negg": [0.43444502353668213, 0.5356224179267883, 0.6728551387786865], "cos_apical_negg": [0.11481891572475433, 0.20972947776317596, 0.2915850877761841], "cos_Ac_negg": [0.4344581663608551, 0.5355129837989807, 0.6728971004486084], "r_norm": [1.0446279048919678, 1.040521264076233, 1.038030743598938]}, {"step": 2800, "epoch": 5, "train_loss": 0.08406370878219604, "cos_r_negg": [0.11313486844301224, 0.15717515349388123, 0.3214162588119507], "cos_innovation_negg": [0.4879230856895447, 0.582575798034668, 0.6949892044067383], "cos_apical_negg": [0.11313486844301224, 0.15717515349388123, 0.3214162588119507], "cos_Ac_negg": [0.4878532290458679, 0.5824505090713501, 0.6949414014816284], "r_norm": [1.0404664278030396, 1.0348474979400635, 1.0308709144592285]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9608, "test_loss": 0.1572404094696045}, {"step": 2900, "epoch": 6, "train_loss": 0.023374833166599274, "cos_r_negg": [0.08860492706298828, 0.15452654659748077, 0.2529580593109131], "cos_innovation_negg": [0.4613765478134155, 0.5762410163879395, 0.6754223704338074], "cos_apical_negg": [0.08860492706298828, 0.15452654659748077, 0.2529580593109131], "cos_Ac_negg": [0.4612863063812256, 0.5760371685028076, 0.6753679513931274], "r_norm": [1.0332884788513184, 1.0295827388763428, 1.0252538919448853]}, {"step": 3000, "epoch": 6, "train_loss": 0.25347787141799927, "cos_r_negg": [0.08315202593803406, 0.1285639852285385, 0.20180664956569672], "cos_innovation_negg": [0.41670238971710205, 0.4907757043838501, 0.6787306666374207], "cos_apical_negg": [0.08315202593803406, 0.1285639852285385, 0.20180664956569672], "cos_Ac_negg": [0.4165520966053009, 0.4903995394706726, 0.6787961721420288], "r_norm": [1.0317394733428955, 1.0286509990692139, 1.022667646408081], "ldr": {"dL_sdil": -0.006295233964920044, "dL_bp": -0.006876260042190552, "ratio": 0.9155026026203902}}, {"step": 3100, "epoch": 6, "train_loss": 0.09500887989997864, "cos_r_negg": [0.060014963150024414, 0.15525293350219727, 0.2267547845840454], "cos_innovation_negg": [0.42980897426605225, 0.5274608135223389, 0.6904079914093018], "cos_apical_negg": [0.060014963150024414, 0.15525293350219727, 0.2267547845840454], "cos_Ac_negg": [0.4297827482223511, 0.5274139046669006, 0.6906270980834961], "r_norm": [1.0476242303848267, 1.0429370403289795, 1.0376466512680054]}, {"step": 3200, "epoch": 6, "train_loss": 0.10645434260368347, "cos_r_negg": [0.0909779965877533, 0.12043160200119019, 0.22367516160011292], "cos_innovation_negg": [0.45399796962738037, 0.5186017751693726, 0.7197713851928711], "cos_apical_negg": [0.0909779965877533, 0.12043160200119019, 0.22367516160011292], "cos_Ac_negg": [0.4539446234703064, 0.518485426902771, 0.7198947072029114], "r_norm": [1.0119233131408691, 1.007584571838379, 1.0031044483184814]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9634, "test_loss": 0.14686290378570557}, {"step": 3300, "epoch": 7, "train_loss": 0.054971843957901, "cos_r_negg": [0.0789458379149437, 0.1460288166999817, 0.25119268894195557], "cos_innovation_negg": [0.4534943997859955, 0.5249248743057251, 0.7382019758224487], "cos_apical_negg": [0.0789458379149437, 0.1460288166999817, 0.25119268894195557], "cos_Ac_negg": [0.45348697900772095, 0.5246008038520813, 0.7382972240447998], "r_norm": [1.0166678428649902, 1.0093374252319336, 1.007851004600525]}, {"step": 3400, "epoch": 7, "train_loss": 0.10781344026327133, "cos_r_negg": [0.07544979453086853, 0.12924110889434814, 0.18920454382896423], "cos_innovation_negg": [0.4369913637638092, 0.5438998937606812, 0.7515773773193359], "cos_apical_negg": [0.07544979453086853, 0.12924110889434814, 0.18920454382896423], "cos_Ac_negg": [0.43686461448669434, 0.5438346266746521, 0.7518844604492188], "r_norm": [1.0381009578704834, 1.0323768854141235, 1.0269943475723267]}, {"step": 3500, "epoch": 7, "train_loss": 0.0555838942527771, "cos_r_negg": [0.050943776965141296, 0.12314064055681229, 0.24524903297424316], "cos_innovation_negg": [0.424956738948822, 0.5067443251609802, 0.7101840972900391], "cos_apical_negg": [0.050943776965141296, 0.12314064055681229, 0.24524903297424316], "cos_Ac_negg": [0.42496949434280396, 0.5064550638198853, 0.7104209661483765], "r_norm": [1.0190849304199219, 1.013673186302185, 1.0090439319610596], "ldr": {"dL_sdil": -0.0051877498626708984, "dL_bp": -0.00829671323299408, "ratio": 0.6252777114243789}}, {"step": 3600, "epoch": 7, "train_loss": 0.016231929883360863, "cos_r_negg": [0.04102472960948944, 0.11987172067165375, 0.23962877690792084], "cos_innovation_negg": [0.39390552043914795, 0.4806842803955078, 0.708020806312561], "cos_apical_negg": [0.04102472960948944, 0.11987172067165375, 0.23962877690792084], "cos_Ac_negg": [0.3938722014427185, 0.4802889823913574, 0.7081338167190552], "r_norm": [1.033786654472351, 1.0289814472198486, 1.024149775505066]}, {"step": 3700, "epoch": 7, "train_loss": 0.07463322579860687, "cos_r_negg": [0.06463445723056793, 0.16267982125282288, 0.2788849472999573], "cos_innovation_negg": [0.4540909230709076, 0.5231131315231323, 0.7329192161560059], "cos_apical_negg": [0.06463445723056793, 0.16267982125282288, 0.2788849472999573], "cos_Ac_negg": [0.4540967345237732, 0.5229589939117432, 0.733177125453949], "r_norm": [1.0419820547103882, 1.0341936349868774, 1.0294487476348877]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.964, "test_loss": 0.15205982093811035}, {"step": 3800, "epoch": 8, "train_loss": 0.02382911555469036, "cos_r_negg": [0.06859715282917023, 0.11703038960695267, 0.27915629744529724], "cos_innovation_negg": [0.4253789186477661, 0.5059475302696228, 0.7277424335479736], "cos_apical_negg": [0.06859715282917023, 0.11703038960695267, 0.27915629744529724], "cos_Ac_negg": [0.4253871440887451, 0.5056382417678833, 0.7274767756462097], "r_norm": [1.0048519372940063, 0.999269962310791, 0.9951889514923096]}, {"step": 3900, "epoch": 8, "train_loss": 0.1512923240661621, "cos_r_negg": [0.061481766402721405, 0.12657298147678375, 0.21981723606586456], "cos_innovation_negg": [0.41390371322631836, 0.5124120712280273, 0.7214871048927307], "cos_apical_negg": [0.061481766402721405, 0.12657298147678375, 0.21981723606586456], "cos_Ac_negg": [0.41383153200149536, 0.5121541023254395, 0.7217346429824829], "r_norm": [1.024421215057373, 1.0178453922271729, 1.014369249343872]}, {"step": 4000, "epoch": 8, "train_loss": 0.06986392289400101, "cos_r_negg": [0.06009560823440552, 0.13233833014965057, 0.17223654687404633], "cos_innovation_negg": [0.41721194982528687, 0.5196924209594727, 0.7315075397491455], "cos_apical_negg": [0.06009560823440552, 0.13233833014965057, 0.17223654687404633], "cos_Ac_negg": [0.4172259569168091, 0.5191183090209961, 0.7319979071617126], "r_norm": [1.0141053199768066, 1.0076427459716797, 1.003874659538269], "ldr": {"dL_sdil": -0.00461287796497345, "dL_bp": -0.007200658321380615, "ratio": 0.6406189210890098}}, {"step": 4100, "epoch": 8, "train_loss": 0.04138413816690445, "cos_r_negg": [0.0579494871199131, 0.11536955088376999, 0.24498414993286133], "cos_innovation_negg": [0.4185745418071747, 0.4919985830783844, 0.7416543960571289], "cos_apical_negg": [0.0579494871199131, 0.11536955088376999, 0.24498414993286133], "cos_Ac_negg": [0.41827186942100525, 0.49132803082466125, 0.7418818473815918], "r_norm": [1.024757742881775, 1.0176424980163574, 1.01226007938385]}, {"step": 4200, "epoch": 8, "train_loss": 0.08453892171382904, "cos_r_negg": [0.05851001292467117, 0.12678737938404083, 0.2562369108200073], "cos_innovation_negg": [0.432187020778656, 0.5198979377746582, 0.7205055952072144], "cos_apical_negg": [0.05851001292467117, 0.12678737938404083, 0.2562369108200073], "cos_Ac_negg": [0.4316670894622803, 0.5195059776306152, 0.7205809950828552], "r_norm": [1.0131231546401978, 1.0065301656723022, 1.0030956268310547]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9622, "test_loss": 0.1646196990966797}, {"step": 4300, "epoch": 9, "train_loss": 0.04316658526659012, "cos_r_negg": [0.054280661046504974, 0.10196350514888763, 0.2806333303451538], "cos_innovation_negg": [0.4090093970298767, 0.5053075551986694, 0.7531549334526062], "cos_apical_negg": [0.054280661046504974, 0.10196350514888763, 0.2806333303451538], "cos_Ac_negg": [0.40875399112701416, 0.5050163269042969, 0.7533770203590393], "r_norm": [1.0165133476257324, 1.0105763673782349, 1.005072832107544]}, {"step": 4400, "epoch": 9, "train_loss": 0.044569067656993866, "cos_r_negg": [0.04354625195264816, 0.10968311876058578, 0.2889139652252197], "cos_innovation_negg": [0.4083552956581116, 0.48645853996276855, 0.7443960905075073], "cos_apical_negg": [0.04354625195264816, 0.10968311876058578, 0.2889139652252197], "cos_Ac_negg": [0.4082323908805847, 0.4860506057739258, 0.7447205781936646], "r_norm": [1.006726861000061, 1.0011639595031738, 0.997625470161438]}, {"step": 4500, "epoch": 9, "train_loss": 0.13628464937210083, "cos_r_negg": [0.039948031306266785, 0.12244822829961777, 0.24630369246006012], "cos_innovation_negg": [0.41494235396385193, 0.4948727786540985, 0.740593433380127], "cos_apical_negg": [0.039948031306266785, 0.12244822829961777, 0.24630369246006012], "cos_Ac_negg": [0.41452139616012573, 0.4943540096282959, 0.7411041259765625], "r_norm": [1.0156197547912598, 1.009873390197754, 1.0053232908248901], "ldr": {"dL_sdil": -0.004762277007102966, "dL_bp": -0.007550597190856934, "ratio": 0.6307152781066957}}, {"step": 4600, "epoch": 9, "train_loss": 0.021538682281970978, "cos_r_negg": [0.0696280226111412, 0.13792680203914642, 0.3534112274646759], "cos_innovation_negg": [0.40972524881362915, 0.4834335744380951, 0.7794768810272217], "cos_apical_negg": [0.0696280226111412, 0.13792680203914642, 0.3534112274646759], "cos_Ac_negg": [0.40946805477142334, 0.48318007588386536, 0.7795393466949463], "r_norm": [1.04246187210083, 1.0343941450119019, 1.0295746326446533]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9619, "test_loss": 0.17044685344696045}, {"step": 4700, "epoch": 10, "train_loss": 0.08341549336910248, "cos_r_negg": [0.04571884498000145, 0.11891037225723267, 0.30167603492736816], "cos_innovation_negg": [0.38895994424819946, 0.47783076763153076, 0.7742821574211121], "cos_apical_negg": [0.04571884498000145, 0.11891037225723267, 0.30167603492736816], "cos_Ac_negg": [0.3886677026748657, 0.4773964285850525, 0.7738631367683411], "r_norm": [1.0328233242034912, 1.0247845649719238, 1.0186296701431274]}, {"step": 4800, "epoch": 10, "train_loss": 0.050148431211709976, "cos_r_negg": [0.02763083577156067, 0.10058742761611938, 0.3184281289577484], "cos_innovation_negg": [0.38614290952682495, 0.48786497116088867, 0.7830615043640137], "cos_apical_negg": [0.02763083577156067, 0.10058742761611938, 0.3184281289577484], "cos_Ac_negg": [0.3859814405441284, 0.48769450187683105, 0.7839152216911316], "r_norm": [1.0237292051315308, 1.0161826610565186, 1.0115747451782227]}, {"step": 4900, "epoch": 10, "train_loss": 0.055147189646959305, "cos_r_negg": [0.03308366984128952, 0.12296850979328156, 0.22171971201896667], "cos_innovation_negg": [0.37891247868537903, 0.4514099955558777, 0.7571605443954468], "cos_apical_negg": [0.03308366984128952, 0.12296850979328156, 0.22171971201896667], "cos_Ac_negg": [0.37868732213974, 0.451760470867157, 0.7571224570274353], "r_norm": [1.0504040718078613, 1.0420150756835938, 1.0372602939605713]}, {"step": 5000, "epoch": 10, "train_loss": 0.01705363765358925, "cos_r_negg": [0.054442863911390305, 0.11831051856279373, 0.326398104429245], "cos_innovation_negg": [0.4010009467601776, 0.4879360496997833, 0.7031821608543396], "cos_apical_negg": [0.054442863911390305, 0.11831051856279373, 0.326398104429245], "cos_Ac_negg": [0.40074652433395386, 0.48770245909690857, 0.7030981779098511], "r_norm": [1.0476109981536865, 1.0388975143432617, 1.0337738990783691], "ldr": {"dL_sdil": -0.007159620523452759, "dL_bp": -0.01317286491394043, "ratio": 0.5435127870988761}}, {"step": 5100, "epoch": 10, "train_loss": 0.09889398515224457, "cos_r_negg": [0.04644930362701416, 0.12823310494422913, 0.316743940114975], "cos_innovation_negg": [0.370052307844162, 0.4791232943534851, 0.779529333114624], "cos_apical_negg": [0.04644930362701416, 0.12823310494422913, 0.316743940114975], "cos_Ac_negg": [0.3699507713317871, 0.4789623022079468, 0.7796865105628967], "r_norm": [1.012260913848877, 1.0053958892822266, 1.0003650188446045]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.965, "test_loss": 0.15641935958862305}, {"step": 5200, "epoch": 11, "train_loss": 0.011618003249168396, "cos_r_negg": [0.029473181813955307, 0.13050757348537445, 0.2541808485984802], "cos_innovation_negg": [0.3835332691669464, 0.4815676808357239, 0.7470118403434753], "cos_apical_negg": [0.029473181813955307, 0.13050757348537445, 0.2541808485984802], "cos_Ac_negg": [0.3832237720489502, 0.48124536871910095, 0.7483774423599243], "r_norm": [1.0235294103622437, 1.0155537128448486, 1.0106277465820312]}, {"step": 5300, "epoch": 11, "train_loss": 0.06883976608514786, "cos_r_negg": [0.02978581003844738, 0.1577966809272766, 0.26428189873695374], "cos_innovation_negg": [0.3612690269947052, 0.44747382402420044, 0.7369123697280884], "cos_apical_negg": [0.02978581003844738, 0.1577966809272766, 0.26428189873695374], "cos_Ac_negg": [0.36127275228500366, 0.4472388029098511, 0.736887514591217], "r_norm": [1.0157907009124756, 1.0084893703460693, 1.003711462020874]}, {"step": 5400, "epoch": 11, "train_loss": 0.09158950299024582, "cos_r_negg": [0.048825714737176895, 0.1291065514087677, 0.19313271343708038], "cos_innovation_negg": [0.36108851432800293, 0.45001083612442017, 0.7426520586013794], "cos_apical_negg": [0.048825714737176895, 0.1291065514087677, 0.19313271343708038], "cos_Ac_negg": [0.3607645332813263, 0.4500146508216858, 0.7427655458450317], "r_norm": [0.9970957636833191, 0.9907493591308594, 0.9866862297058105]}, {"step": 5500, "epoch": 11, "train_loss": 0.09621293097734451, "cos_r_negg": [0.02780434675514698, 0.1293172538280487, 0.28517085313796997], "cos_innovation_negg": [0.3989107012748718, 0.4655280113220215, 0.7712547779083252], "cos_apical_negg": [0.02780434675514698, 0.1293172538280487, 0.28517085313796997], "cos_Ac_negg": [0.3988367021083832, 0.46450167894363403, 0.7711506485939026], "r_norm": [1.0169309377670288, 1.0087089538574219, 1.003431797027588], "ldr": {"dL_sdil": -0.00443209707736969, "dL_bp": -0.007188484072685242, "ratio": 0.6165551780535684}}, {"step": 5600, "epoch": 11, "train_loss": 0.009339344687759876, "cos_r_negg": [0.02955637313425541, 0.12707480788230896, 0.3031397759914398], "cos_innovation_negg": [0.3965030312538147, 0.4689267575740814, 0.7651573419570923], "cos_apical_negg": [0.02955637313425541, 0.12707480788230896, 0.3031397759914398], "cos_Ac_negg": [0.3964735269546509, 0.4691050052642822, 0.7656296491622925], "r_norm": [1.0046048164367676, 0.9975359439849854, 0.9935814142227173]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9646, "test_loss": 0.1676767059326172}, {"step": 5700, "epoch": 12, "train_loss": 0.039681173861026764, "cos_r_negg": [0.023674480617046356, 0.1559143364429474, 0.2809804081916809], "cos_innovation_negg": [0.3727792501449585, 0.48782408237457275, 0.7735583782196045], "cos_apical_negg": [0.023674480617046356, 0.1559143364429474, 0.2809804081916809], "cos_Ac_negg": [0.3725578486919403, 0.487379252910614, 0.7736832499504089], "r_norm": [1.0121532678604126, 1.004992961883545, 1.000563144683838]}, {"step": 5800, "epoch": 12, "train_loss": 0.009409253485500813, "cos_r_negg": [0.012268457561731339, 0.12750187516212463, 0.31009548902511597], "cos_innovation_negg": [0.3641427159309387, 0.45033109188079834, 0.7750030755996704], "cos_apical_negg": [0.012268457561731339, 0.12750187516212463, 0.31009548902511597], "cos_Ac_negg": [0.36390671133995056, 0.44966304302215576, 0.7762756943702698], "r_norm": [1.0208228826522827, 1.0131936073303223, 1.0075652599334717]}, {"step": 5900, "epoch": 12, "train_loss": 0.03900974616408348, "cos_r_negg": [0.037060368806123734, 0.11824866384267807, 0.3086228370666504], "cos_innovation_negg": [0.3732796907424927, 0.4792660176753998, 0.7758967876434326], "cos_apical_negg": [0.037060368806123734, 0.11824866384267807, 0.3086228370666504], "cos_Ac_negg": [0.3726726174354553, 0.4773159623146057, 0.7757237553596497], "r_norm": [1.0238292217254639, 1.015966773033142, 1.0104520320892334]}, {"step": 6000, "epoch": 12, "train_loss": 0.00767051987349987, "cos_r_negg": [0.026138661429286003, 0.11731883883476257, 0.30830878019332886], "cos_innovation_negg": [0.36157429218292236, 0.4710817337036133, 0.7574659585952759], "cos_apical_negg": [0.026138661429286003, 0.11731883883476257, 0.30830878019332886], "cos_Ac_negg": [0.36157506704330444, 0.47078239917755127, 0.7574666738510132], "r_norm": [1.0263004302978516, 1.0175118446350098, 1.012162446975708], "ldr": {"dL_sdil": -0.005875617265701294, "dL_bp": -0.01049591600894928, "ratio": 0.5598003319282933}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9656, "test_loss": 0.16640939407348632}, {"step": 6100, "epoch": 13, "train_loss": 0.013630226254463196, "cos_r_negg": [0.018729209899902344, 0.13422933220863342, 0.26772210001945496], "cos_innovation_negg": [0.3898214101791382, 0.4603390693664551, 0.7649310827255249], "cos_apical_negg": [0.018729209899902344, 0.13422933220863342, 0.26772210001945496], "cos_Ac_negg": [0.38953709602355957, 0.4590319097042084, 0.764623761177063], "r_norm": [1.0028607845306396, 0.9960696697235107, 0.9908932447433472]}, {"step": 6200, "epoch": 13, "train_loss": 0.014845678582787514, "cos_r_negg": [0.04792290925979614, 0.13098245859146118, 0.2532523572444916], "cos_innovation_negg": [0.3858796954154968, 0.4564515948295593, 0.7580298185348511], "cos_apical_negg": [0.04792290925979614, 0.13098245859146118, 0.2532523572444916], "cos_Ac_negg": [0.38527148962020874, 0.45281267166137695, 0.7577612996101379], "r_norm": [0.9996435046195984, 0.9916858673095703, 0.9880802631378174]}, {"step": 6300, "epoch": 13, "train_loss": 0.030698556452989578, "cos_r_negg": [0.024974389001727104, 0.1229492723941803, 0.23045440018177032], "cos_innovation_negg": [0.3792564272880554, 0.44726669788360596, 0.7672052979469299], "cos_apical_negg": [0.024974389001727104, 0.1229492723941803, 0.23045440018177032], "cos_Ac_negg": [0.37906143069267273, 0.4448671042919159, 0.7671007513999939], "r_norm": [0.9923502802848816, 0.9854509830474854, 0.9807218313217163]}, {"step": 6400, "epoch": 13, "train_loss": 0.04797133058309555, "cos_r_negg": [0.031597599387168884, 0.12947042286396027, 0.2632352113723755], "cos_innovation_negg": [0.3592355251312256, 0.45848363637924194, 0.755677342414856], "cos_apical_negg": [0.031597599387168884, 0.12947042286396027, 0.2632352113723755], "cos_Ac_negg": [0.3589644432067871, 0.45814529061317444, 0.7558613419532776], "r_norm": [0.9953324794769287, 0.9877307415008545, 0.9834027290344238]}, {"step": 6500, "epoch": 13, "train_loss": 0.003274256829172373, "cos_r_negg": [0.012578103691339493, 0.11179740726947784, 0.2535165250301361], "cos_innovation_negg": [0.3779268264770508, 0.4656335413455963, 0.7351794838905334], "cos_apical_negg": [0.012578103691339493, 0.11179740726947784, 0.2535165250301361], "cos_Ac_negg": [0.3777846097946167, 0.4647585153579712, 0.7358333468437195], "r_norm": [0.9992240071296692, 0.9908066391944885, 0.9862110614776611], "ldr": {"dL_sdil": -0.004559755325317383, "dL_bp": -0.008030295372009277, "ratio": 0.5678191291955524}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9639, "test_loss": 0.17209006233215332}, {"step": 6600, "epoch": 14, "train_loss": 0.007514375727623701, "cos_r_negg": [0.01508256234228611, 0.13311469554901123, 0.27300024032592773], "cos_innovation_negg": [0.38066548109054565, 0.46259987354278564, 0.7482706308364868], "cos_apical_negg": [0.01508256234228611, 0.13311469554901123, 0.27300024032592773], "cos_Ac_negg": [0.37996968626976013, 0.4604552388191223, 0.7488212585449219], "r_norm": [1.0007352828979492, 0.9943813681602478, 0.9901024103164673]}, {"step": 6700, "epoch": 14, "train_loss": 0.06227608770132065, "cos_r_negg": [0.00989779643714428, 0.14579452574253082, 0.3493391275405884], "cos_innovation_negg": [0.38116636872291565, 0.45595836639404297, 0.780990719795227], "cos_apical_negg": [0.00989779643714428, 0.14579452574253082, 0.3493391275405884], "cos_Ac_negg": [0.380575954914093, 0.4534801244735718, 0.7811557054519653], "r_norm": [1.0045349597930908, 0.9969815611839294, 0.9917994737625122]}, {"step": 6800, "epoch": 14, "train_loss": 0.06896437704563141, "cos_r_negg": [0.012832611799240112, 0.15619882941246033, 0.32386553287506104], "cos_innovation_negg": [0.3918267488479614, 0.4652990698814392, 0.7924189567565918], "cos_apical_negg": [0.012832611799240112, 0.15619882941246033, 0.32386553287506104], "cos_Ac_negg": [0.39119476079940796, 0.4643750488758087, 0.7919941544532776], "r_norm": [1.0126630067825317, 1.0044729709625244, 0.9982580542564392]}, {"step": 6900, "epoch": 14, "train_loss": 0.02285294607281685, "cos_r_negg": [-0.0004212576895952225, 0.13315272331237793, 0.2762680947780609], "cos_innovation_negg": [0.3808654546737671, 0.49239078164100647, 0.7823525667190552], "cos_apical_negg": [-0.0004212576895952225, 0.13315272331237793, 0.2762680947780609], "cos_Ac_negg": [0.38029754161834717, 0.49130237102508545, 0.7821752429008484], "r_norm": [0.9951581954956055, 0.9866114854812622, 0.9817646741867065]}, {"step": 7000, "epoch": 14, "train_loss": 0.052804600447416306, "cos_r_negg": [0.03642439842224121, 0.10317088663578033, 0.24736720323562622], "cos_innovation_negg": [0.36583250761032104, 0.4374212622642517, 0.7792471647262573], "cos_apical_negg": [0.03642439842224121, 0.10317088663578033, 0.24736720323562622], "cos_Ac_negg": [0.36550819873809814, 0.4369518756866455, 0.7794103622436523], "r_norm": [1.0036711692810059, 0.995650053024292, 0.9915581941604614], "ldr": {"dL_sdil": -0.003786906599998474, "dL_bp": -0.006322965025901794, "ratio": 0.5989131023950868}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.965, "test_loss": 0.17623918800354005}], "final": {"test_acc": 0.965, "test_loss": 0.17623918800354005, "wall_s": 27.3532133102417, "cos_r_negg": [0.02815653569996357, 0.10895447432994843, 0.2546684443950653], "cos_innovation_negg": [0.3487493693828583, 0.4369896352291107, 0.7721617221832275], "cos_apical_negg": [0.02815653569996357, 0.10895447432994843, 0.2546684443950653], "cos_Ac_negg": [0.3478301167488098, 0.4360348582267761, 0.77239590883255]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_raw_s1.json b/results/nuis_ctrl_v1_rho0p05_raw_s1.json new file mode 100644 index 0000000..8f3ba47 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_raw_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_raw_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.013905191794037819, -0.013632269576191902, -0.006711989641189575], "cos_innovation_negg": [0.014363561756908894, -0.013818755745887756, -0.0063642337918281555], "cos_apical_negg": [0.013905191794037819, -0.013632269576191902, -0.006711989641189575], "cos_Ac_negg": [0.014363492839038372, -0.0138187725096941, -0.0063642594031989574], "r_norm": [4.815765857696533, 4.82281494140625, 4.805631637573242], "ldr": {"dL_sdil": -0.0417790412902832, "dL_bp": -0.21432781219482422, "ratio": 0.19493056389856678}}, {"step": 100, "epoch": 0, "train_loss": 0.3111538290977478, "cos_r_negg": [0.22585797309875488, 0.2289954423904419, 0.22102117538452148], "cos_innovation_negg": [0.47757360339164734, 0.41555705666542053, 0.31677794456481934], "cos_apical_negg": [0.22585797309875488, 0.2289954423904419, 0.22102117538452148], "cos_Ac_negg": [0.4775737524032593, 0.4155714511871338, 0.31678393483161926], "r_norm": [1.507110357284546, 1.5036137104034424, 1.5179942846298218]}, {"step": 200, "epoch": 0, "train_loss": 0.31935569643974304, "cos_r_negg": [0.21915887296199799, 0.2109759896993637, 0.24562522768974304], "cos_innovation_negg": [0.45117610692977905, 0.3945627808570862, 0.34682363271713257], "cos_apical_negg": [0.21915887296199799, 0.2109759896993637, 0.24562522768974304], "cos_Ac_negg": [0.45115044713020325, 0.3945918381214142, 0.3468201756477356], "r_norm": [1.3864176273345947, 1.3908636569976807, 1.404003620147705]}, {"step": 300, "epoch": 0, "train_loss": 0.2104194015264511, "cos_r_negg": [0.16776320338249207, 0.19028759002685547, 0.23572096228599548], "cos_innovation_negg": [0.4569571614265442, 0.4386172890663147, 0.4506438076496124], "cos_apical_negg": [0.16776320338249207, 0.19028759002685547, 0.23572096228599548], "cos_Ac_negg": [0.45695632696151733, 0.43851780891418457, 0.4506588280200958], "r_norm": [1.2572340965270996, 1.2549448013305664, 1.2671852111816406]}, {"step": 400, "epoch": 0, "train_loss": 0.18861497938632965, "cos_r_negg": [0.16798993945121765, 0.2042803317308426, 0.27786386013031006], "cos_innovation_negg": [0.5486142635345459, 0.5082403421401978, 0.4526609778404236], "cos_apical_negg": [0.16798993945121765, 0.2042803317308426, 0.27786386013031006], "cos_Ac_negg": [0.5486135482788086, 0.5081335306167603, 0.4527098834514618], "r_norm": [1.2299907207489014, 1.2317063808441162, 1.2414379119873047]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9422, "test_loss": 0.20943933486938476}, {"step": 500, "epoch": 1, "train_loss": 0.19546829164028168, "cos_r_negg": [0.12399314343929291, 0.19445103406906128, 0.2384604513645172], "cos_innovation_negg": [0.49418824911117554, 0.486931174993515, 0.4326460361480713], "cos_apical_negg": [0.12399314343929291, 0.19445103406906128, 0.2384604513645172], "cos_Ac_negg": [0.494186669588089, 0.4869386553764343, 0.43236231803894043], "r_norm": [1.1982481479644775, 1.1957364082336426, 1.206697940826416], "ldr": {"dL_sdil": -0.00836414098739624, "dL_bp": -0.006997942924499512, "ratio": 1.1952285232441273}}, {"step": 600, "epoch": 1, "train_loss": 0.10189378261566162, "cos_r_negg": [0.12711183726787567, 0.21554352343082428, 0.2960144877433777], "cos_innovation_negg": [0.49261781573295593, 0.517648458480835, 0.4768499433994293], "cos_apical_negg": [0.12711183726787567, 0.21554352343082428, 0.2960144877433777], "cos_Ac_negg": [0.49261361360549927, 0.5176858901977539, 0.4766700267791748], "r_norm": [1.1861131191253662, 1.1880137920379639, 1.1966147422790527]}, {"step": 700, "epoch": 1, "train_loss": 0.1639467477798462, "cos_r_negg": [0.16854949295520782, 0.21466293931007385, 0.405530720949173], "cos_innovation_negg": [0.5290984511375427, 0.5688223838806152, 0.6095681190490723], "cos_apical_negg": [0.16854949295520782, 0.21466293931007385, 0.405530720949173], "cos_Ac_negg": [0.5290928483009338, 0.5687499046325684, 0.6095190048217773], "r_norm": [1.2222474813461304, 1.225698471069336, 1.2332630157470703]}, {"step": 800, "epoch": 1, "train_loss": 0.1482793390750885, "cos_r_negg": [0.14247897267341614, 0.20840352773666382, 0.2946995496749878], "cos_innovation_negg": [0.49229753017425537, 0.5285365581512451, 0.5543482303619385], "cos_apical_negg": [0.14247897267341614, 0.20840352773666382, 0.2946995496749878], "cos_Ac_negg": [0.4923862814903259, 0.5285008549690247, 0.5543349981307983], "r_norm": [1.1485624313354492, 1.151463508605957, 1.1598212718963623]}, {"step": 900, "epoch": 1, "train_loss": 0.24120372533798218, "cos_r_negg": [0.15464136004447937, 0.22585511207580566, 0.2914172410964966], "cos_innovation_negg": [0.526918888092041, 0.5539251565933228, 0.5657846927642822], "cos_apical_negg": [0.15464136004447937, 0.22585511207580566, 0.2914172410964966], "cos_Ac_negg": [0.5269187092781067, 0.5538864135742188, 0.5658643245697021], "r_norm": [1.1452395915985107, 1.1458568572998047, 1.1549901962280273]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9489, "test_loss": 0.19619377784729003}, {"step": 1000, "epoch": 2, "train_loss": 0.05176971107721329, "cos_r_negg": [0.11876438558101654, 0.20083294808864594, 0.28751203417778015], "cos_innovation_negg": [0.45458948612213135, 0.49184131622314453, 0.49922239780426025], "cos_apical_negg": [0.11876438558101654, 0.20083294808864594, 0.28751203417778015], "cos_Ac_negg": [0.4545850157737732, 0.4917981028556824, 0.4991696774959564], "r_norm": [1.1385282278060913, 1.1386324167251587, 1.14955735206604], "ldr": {"dL_sdil": -0.006788343191146851, "dL_bp": -0.006677001714706421, "ratio": 1.0166753703530127}}, {"step": 1100, "epoch": 2, "train_loss": 0.16464105248451233, "cos_r_negg": [0.11798076331615448, 0.23670539259910583, 0.293226420879364], "cos_innovation_negg": [0.5064412355422974, 0.5673417448997498, 0.5176621675491333], "cos_apical_negg": [0.11798076331615448, 0.23670539259910583, 0.293226420879364], "cos_Ac_negg": [0.5064129829406738, 0.5672239065170288, 0.5174068212509155], "r_norm": [1.1557585000991821, 1.1562490463256836, 1.162174940109253]}, {"step": 1200, "epoch": 2, "train_loss": 0.22122842073440552, "cos_r_negg": [0.11416029930114746, 0.23655816912651062, 0.34759199619293213], "cos_innovation_negg": [0.5368348360061646, 0.5717867612838745, 0.5662579536437988], "cos_apical_negg": [0.11416029930114746, 0.23655816912651062, 0.34759199619293213], "cos_Ac_negg": [0.5368198752403259, 0.5716375112533569, 0.5660883188247681], "r_norm": [1.1090397834777832, 1.1073741912841797, 1.1163573265075684]}, {"step": 1300, "epoch": 2, "train_loss": 0.16772988438606262, "cos_r_negg": [0.1405804306268692, 0.2335662841796875, 0.3327704668045044], "cos_innovation_negg": [0.5201165676116943, 0.570197582244873, 0.5598894953727722], "cos_apical_negg": [0.1405804306268692, 0.2335662841796875, 0.3327704668045044], "cos_Ac_negg": [0.5201260447502136, 0.5701526403427124, 0.5599179267883301], "r_norm": [1.1047096252441406, 1.1055306196212769, 1.1118117570877075]}, {"step": 1400, "epoch": 2, "train_loss": 0.13406571745872498, "cos_r_negg": [0.13619279861450195, 0.20941613614559174, 0.3439789116382599], "cos_innovation_negg": [0.5422729253768921, 0.591759979724884, 0.6170070171356201], "cos_apical_negg": [0.13619279861450195, 0.20941613614559174, 0.3439789116382599], "cos_Ac_negg": [0.5422627329826355, 0.5916441679000854, 0.6169291734695435], "r_norm": [1.0734968185424805, 1.0720207691192627, 1.0773144960403442]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9577, "test_loss": 0.1640996253967285}, {"step": 1500, "epoch": 3, "train_loss": 0.10332301259040833, "cos_r_negg": [0.11957291513681412, 0.20736415684223175, 0.30939364433288574], "cos_innovation_negg": [0.5052887201309204, 0.5775151252746582, 0.5726076364517212], "cos_apical_negg": [0.11957291513681412, 0.20736415684223175, 0.30939364433288574], "cos_Ac_negg": [0.5052045583724976, 0.5773980617523193, 0.5725935697555542], "r_norm": [1.0954232215881348, 1.096743106842041, 1.1033844947814941], "ldr": {"dL_sdil": -0.004838660359382629, "dL_bp": -0.004917040467262268, "ratio": 0.984059495055716}}, {"step": 1600, "epoch": 3, "train_loss": 0.105115607380867, "cos_r_negg": [0.10056674480438232, 0.17087051272392273, 0.33277279138565063], "cos_innovation_negg": [0.46659916639328003, 0.5546510219573975, 0.5421147346496582], "cos_apical_negg": [0.10056674480438232, 0.17087051272392273, 0.33277279138565063], "cos_Ac_negg": [0.4664914608001709, 0.5545365810394287, 0.5414087772369385], "r_norm": [1.0722692012786865, 1.0694310665130615, 1.0746160745620728]}, {"step": 1700, "epoch": 3, "train_loss": 0.11515215039253235, "cos_r_negg": [0.09613379091024399, 0.22954893112182617, 0.36259105801582336], "cos_innovation_negg": [0.45003971457481384, 0.566016674041748, 0.6306129097938538], "cos_apical_negg": [0.09613379091024399, 0.22954893112182617, 0.36259105801582336], "cos_Ac_negg": [0.4499720335006714, 0.565842866897583, 0.6303963661193848], "r_norm": [1.1150307655334473, 1.116746425628662, 1.1212819814682007]}, {"step": 1800, "epoch": 3, "train_loss": 0.02130863070487976, "cos_r_negg": [0.10759153962135315, 0.216238334774971, 0.3577558994293213], "cos_innovation_negg": [0.5138797760009766, 0.5725353956222534, 0.6255130767822266], "cos_apical_negg": [0.10759153962135315, 0.216238334774971, 0.3577558994293213], "cos_Ac_negg": [0.5136306881904602, 0.5721944570541382, 0.6250292062759399], "r_norm": [1.049177646636963, 1.0476912260055542, 1.055837869644165]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9571, "test_loss": 0.16132894821166993}, {"step": 1900, "epoch": 4, "train_loss": 0.08336880803108215, "cos_r_negg": [0.11278876662254333, 0.18105575442314148, 0.34281179308891296], "cos_innovation_negg": [0.48743218183517456, 0.5750161409378052, 0.6512165665626526], "cos_apical_negg": [0.11278876662254333, 0.18105575442314148, 0.34281179308891296], "cos_Ac_negg": [0.48735135793685913, 0.5750393271446228, 0.6512870788574219], "r_norm": [1.0934102535247803, 1.0947078466415405, 1.1014515161514282]}, {"step": 2000, "epoch": 4, "train_loss": 0.14404454827308655, "cos_r_negg": [0.09042025357484818, 0.18417401611804962, 0.3266880512237549], "cos_innovation_negg": [0.46254539489746094, 0.5401381850242615, 0.5898675918579102], "cos_apical_negg": [0.09042025357484818, 0.18417401611804962, 0.3266880512237549], "cos_Ac_negg": [0.4624313712120056, 0.5399748086929321, 0.5898659229278564], "r_norm": [1.0635663270950317, 1.0584098100662231, 1.0648940801620483], "ldr": {"dL_sdil": -0.005668938159942627, "dL_bp": -0.0066918134689331055, "ratio": 0.8471452747840028}}, {"step": 2100, "epoch": 4, "train_loss": 0.06375595927238464, "cos_r_negg": [0.12302787601947784, 0.2330954372882843, 0.4018273949623108], "cos_innovation_negg": [0.4991646111011505, 0.5582159757614136, 0.6183664202690125], "cos_apical_negg": [0.12302787601947784, 0.2330954372882843, 0.4018273949623108], "cos_Ac_negg": [0.4991351366043091, 0.5577704906463623, 0.6179418563842773], "r_norm": [1.0973044633865356, 1.100116491317749, 1.1074888706207275]}, {"step": 2200, "epoch": 4, "train_loss": 0.03386557847261429, "cos_r_negg": [0.08939173817634583, 0.17289167642593384, 0.2986927330493927], "cos_innovation_negg": [0.48034992814064026, 0.5516642332077026, 0.5839909315109253], "cos_apical_negg": [0.08939173817634583, 0.17289167642593384, 0.2986927330493927], "cos_Ac_negg": [0.48025286197662354, 0.5514553785324097, 0.5833542346954346], "r_norm": [1.0268346071243286, 1.0239297151565552, 1.0311834812164307]}, {"step": 2300, "epoch": 4, "train_loss": 0.18919767439365387, "cos_r_negg": [0.11006394028663635, 0.22932633757591248, 0.3612495958805084], "cos_innovation_negg": [0.5155396461486816, 0.6342329978942871, 0.686772882938385], "cos_apical_negg": [0.11006394028663635, 0.22932633757591248, 0.3612495958805084], "cos_Ac_negg": [0.5153635740280151, 0.6339044570922852, 0.6864478588104248], "r_norm": [1.0438575744628906, 1.0415865182876587, 1.0491456985473633]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9591, "test_loss": 0.16723030738830566}, {"step": 2400, "epoch": 5, "train_loss": 0.12079156190156937, "cos_r_negg": [0.11126136034727097, 0.21823373436927795, 0.3169762194156647], "cos_innovation_negg": [0.4963069260120392, 0.5969429612159729, 0.6623464226722717], "cos_apical_negg": [0.11126136034727097, 0.21823373436927795, 0.3169762194156647], "cos_Ac_negg": [0.49633920192718506, 0.5970313549041748, 0.6623939275741577], "r_norm": [1.0386252403259277, 1.0362870693206787, 1.0430126190185547]}, {"step": 2500, "epoch": 5, "train_loss": 0.05364352464675903, "cos_r_negg": [0.09973915666341782, 0.2178991287946701, 0.3339419662952423], "cos_innovation_negg": [0.5003910064697266, 0.5921323299407959, 0.7173104286193848], "cos_apical_negg": [0.09973915666341782, 0.2178991287946701, 0.3339419662952423], "cos_Ac_negg": [0.500408947467804, 0.5916909575462341, 0.7167575359344482], "r_norm": [1.0779132843017578, 1.071363925933838, 1.0775971412658691], "ldr": {"dL_sdil": -0.009411841630935669, "dL_bp": -0.011350810527801514, "ratio": 0.829177934728385}}, {"step": 2600, "epoch": 5, "train_loss": 0.0950925424695015, "cos_r_negg": [0.07227948307991028, 0.21443215012550354, 0.3914218544960022], "cos_innovation_negg": [0.484333872795105, 0.6018844842910767, 0.6960376501083374], "cos_apical_negg": [0.07227948307991028, 0.21443215012550354, 0.3914218544960022], "cos_Ac_negg": [0.48413121700286865, 0.6007460951805115, 0.695088803768158], "r_norm": [1.037591576576233, 1.0353960990905762, 1.0433489084243774]}, {"step": 2700, "epoch": 5, "train_loss": 0.07270301878452301, "cos_r_negg": [0.07606332004070282, 0.17456433176994324, 0.32549673318862915], "cos_innovation_negg": [0.48064371943473816, 0.5868778228759766, 0.6518148183822632], "cos_apical_negg": [0.07606332004070282, 0.17456433176994324, 0.32549673318862915], "cos_Ac_negg": [0.48056966066360474, 0.5866889953613281, 0.6516287326812744], "r_norm": [1.0150339603424072, 1.0124701261520386, 1.0199952125549316]}, {"step": 2800, "epoch": 5, "train_loss": 0.18392163515090942, "cos_r_negg": [0.07067474722862244, 0.16180387139320374, 0.36166679859161377], "cos_innovation_negg": [0.4727996587753296, 0.5990469455718994, 0.6918765306472778], "cos_apical_negg": [0.07067474722862244, 0.16180387139320374, 0.36166679859161377], "cos_Ac_negg": [0.4728286862373352, 0.599349856376648, 0.6918573379516602], "r_norm": [1.0224902629852295, 1.0210386514663696, 1.0280964374542236]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9622, "test_loss": 0.1542672260284424}, {"step": 2900, "epoch": 6, "train_loss": 0.02426699735224247, "cos_r_negg": [0.08332119882106781, 0.1748197078704834, 0.32856476306915283], "cos_innovation_negg": [0.4771268367767334, 0.610244631767273, 0.6739753484725952], "cos_apical_negg": [0.08332119882106781, 0.1748197078704834, 0.32856476306915283], "cos_Ac_negg": [0.47707369923591614, 0.6102348566055298, 0.6737914085388184], "r_norm": [1.0442066192626953, 1.0395574569702148, 1.047477126121521]}, {"step": 3000, "epoch": 6, "train_loss": 0.3423105478286743, "cos_r_negg": [0.06307509541511536, 0.15948311984539032, 0.30363959074020386], "cos_innovation_negg": [0.46689513325691223, 0.5805035829544067, 0.6361100673675537], "cos_apical_negg": [0.06307509541511536, 0.15948311984539032, 0.30363959074020386], "cos_Ac_negg": [0.4668571949005127, 0.5801676511764526, 0.6358035802841187], "r_norm": [1.0098156929016113, 1.006965160369873, 1.0127813816070557], "ldr": {"dL_sdil": -0.003946930170059204, "dL_bp": -0.0042160749435424805, "ratio": 0.9361622416376848}}, {"step": 3100, "epoch": 6, "train_loss": 0.0975814163684845, "cos_r_negg": [0.07857103645801544, 0.1796930581331253, 0.36494001746177673], "cos_innovation_negg": [0.45325079560279846, 0.5922108888626099, 0.6971831917762756], "cos_apical_negg": [0.07857103645801544, 0.1796930581331253, 0.36494001746177673], "cos_Ac_negg": [0.4532625675201416, 0.5921370983123779, 0.6970545649528503], "r_norm": [1.0387482643127441, 1.0345592498779297, 1.0429034233093262]}, {"step": 3200, "epoch": 6, "train_loss": 0.10800129920244217, "cos_r_negg": [0.03326963260769844, 0.17290058732032776, 0.3843672275543213], "cos_innovation_negg": [0.4869436025619507, 0.5830050706863403, 0.7204266786575317], "cos_apical_negg": [0.03326963260769844, 0.17290058732032776, 0.3843672275543213], "cos_Ac_negg": [0.4869372546672821, 0.582355797290802, 0.7196407318115234], "r_norm": [1.0017832517623901, 0.997281551361084, 1.004207730293274]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9623, "test_loss": 0.1552559108734131}, {"step": 3300, "epoch": 7, "train_loss": 0.05600239336490631, "cos_r_negg": [0.06956698000431061, 0.138072669506073, 0.3201645612716675], "cos_innovation_negg": [0.4550621509552002, 0.5418501496315002, 0.6678740382194519], "cos_apical_negg": [0.06956698000431061, 0.138072669506073, 0.3201645612716675], "cos_Ac_negg": [0.4549746811389923, 0.5419297218322754, 0.6677284240722656], "r_norm": [1.0094234943389893, 1.006321907043457, 1.012253761291504]}, {"step": 3400, "epoch": 7, "train_loss": 0.1245359480381012, "cos_r_negg": [0.0801684632897377, 0.19823044538497925, 0.37976396083831787], "cos_innovation_negg": [0.4689311385154724, 0.605712890625, 0.6607257723808289], "cos_apical_negg": [0.0801684632897377, 0.19823044538497925, 0.37976396083831787], "cos_Ac_negg": [0.46858859062194824, 0.6056188941001892, 0.659482479095459], "r_norm": [1.0204098224639893, 1.0160210132598877, 1.0230128765106201]}, {"step": 3500, "epoch": 7, "train_loss": 0.11146698892116547, "cos_r_negg": [0.07624058425426483, 0.15644685924053192, 0.2621228098869324], "cos_innovation_negg": [0.44706106185913086, 0.49836763739585876, 0.5497217178344727], "cos_apical_negg": [0.07624058425426483, 0.15644685924053192, 0.2621228098869324], "cos_Ac_negg": [0.4469067454338074, 0.4980616867542267, 0.5484806299209595], "r_norm": [1.00234854221344, 0.9983804225921631, 1.0046262741088867], "ldr": {"dL_sdil": -0.003928542137145996, "dL_bp": -0.005212306976318359, "ratio": 0.7537050590064953}}, {"step": 3600, "epoch": 7, "train_loss": 0.02609572932124138, "cos_r_negg": [0.08282971382141113, 0.17203842103481293, 0.3280511498451233], "cos_innovation_negg": [0.477922260761261, 0.5371098518371582, 0.6868182420730591], "cos_apical_negg": [0.08282971382141113, 0.17203842103481293, 0.3280511498451233], "cos_Ac_negg": [0.4778933525085449, 0.5369381308555603, 0.6866243481636047], "r_norm": [1.0103245973587036, 1.0051637887954712, 1.0108013153076172]}, {"step": 3700, "epoch": 7, "train_loss": 0.06168299913406372, "cos_r_negg": [0.07405601441860199, 0.14879007637500763, 0.38227328658103943], "cos_innovation_negg": [0.49088534712791443, 0.5814383029937744, 0.7205300331115723], "cos_apical_negg": [0.07405601441860199, 0.14879007637500763, 0.38227328658103943], "cos_Ac_negg": [0.490772008895874, 0.5812431573867798, 0.7198448181152344], "r_norm": [1.0193647146224976, 1.0127074718475342, 1.01832115650177]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9634, "test_loss": 0.15631410789489747}, {"step": 3800, "epoch": 8, "train_loss": 0.04240557178854942, "cos_r_negg": [0.058022718876600266, 0.1690751612186432, 0.31578749418258667], "cos_innovation_negg": [0.46928760409355164, 0.5878943204879761, 0.6567918658256531], "cos_apical_negg": [0.058022718876600266, 0.1690751612186432, 0.31578749418258667], "cos_Ac_negg": [0.4692709743976593, 0.5876166820526123, 0.6564027070999146], "r_norm": [1.0154097080230713, 1.010310411453247, 1.0156447887420654]}, {"step": 3900, "epoch": 8, "train_loss": 0.13345283269882202, "cos_r_negg": [0.07689585536718369, 0.18118694424629211, 0.2707849442958832], "cos_innovation_negg": [0.4771782159805298, 0.5578058362007141, 0.6736498475074768], "cos_apical_negg": [0.07689585536718369, 0.18118694424629211, 0.2707849442958832], "cos_Ac_negg": [0.4770645797252655, 0.5567095279693604, 0.6727590560913086], "r_norm": [1.02763032913208, 1.0211572647094727, 1.026594638824463]}, {"step": 4000, "epoch": 8, "train_loss": 0.10269910097122192, "cos_r_negg": [0.0932910293340683, 0.16571977734565735, 0.3383203148841858], "cos_innovation_negg": [0.4715930223464966, 0.5827735066413879, 0.679978609085083], "cos_apical_negg": [0.0932910293340683, 0.16571977734565735, 0.3383203148841858], "cos_Ac_negg": [0.47118639945983887, 0.5813443660736084, 0.6783742308616638], "r_norm": [1.0120360851287842, 1.0063525438308716, 1.0133607387542725], "ldr": {"dL_sdil": -0.005438610911369324, "dL_bp": -0.007597222924232483, "ratio": 0.7158682805031372}}, {"step": 4100, "epoch": 8, "train_loss": 0.023356664925813675, "cos_r_negg": [0.0712575763463974, 0.14939527213573456, 0.27680835127830505], "cos_innovation_negg": [0.46732020378112793, 0.6015224456787109, 0.6808948516845703], "cos_apical_negg": [0.0712575763463974, 0.14939527213573456, 0.27680835127830505], "cos_Ac_negg": [0.46705782413482666, 0.6018350720405579, 0.6822611689567566], "r_norm": [0.9958678483963013, 0.9913784861564636, 0.9969359636306763]}, {"step": 4200, "epoch": 8, "train_loss": 0.1036907359957695, "cos_r_negg": [0.07991500198841095, 0.166538268327713, 0.3339913487434387], "cos_innovation_negg": [0.4539138674736023, 0.5708163976669312, 0.6744144558906555], "cos_apical_negg": [0.07991500198841095, 0.166538268327713, 0.3339913487434387], "cos_Ac_negg": [0.4537237584590912, 0.5714869499206543, 0.6745802164077759], "r_norm": [0.9879276156425476, 0.9835188388824463, 0.9894178509712219]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9601, "test_loss": 0.17026060676574706}, {"step": 4300, "epoch": 9, "train_loss": 0.03131965920329094, "cos_r_negg": [0.06058957427740097, 0.15393486618995667, 0.39697808027267456], "cos_innovation_negg": [0.4793262183666229, 0.6009911298751831, 0.7217565774917603], "cos_apical_negg": [0.06058957427740097, 0.15393486618995667, 0.39697808027267456], "cos_Ac_negg": [0.4792332649230957, 0.6009107828140259, 0.7214959859848022], "r_norm": [1.0002045631408691, 0.9942552447319031, 0.9997059106826782]}, {"step": 4400, "epoch": 9, "train_loss": 0.04176610708236694, "cos_r_negg": [0.07677163183689117, 0.146663099527359, 0.3605833053588867], "cos_innovation_negg": [0.44574159383773804, 0.5401622653007507, 0.6500576734542847], "cos_apical_negg": [0.07677163183689117, 0.146663099527359, 0.3605833053588867], "cos_Ac_negg": [0.445831835269928, 0.5404576063156128, 0.647425651550293], "r_norm": [1.017586350440979, 1.0118733644485474, 1.0189217329025269]}, {"step": 4500, "epoch": 9, "train_loss": 0.13476458191871643, "cos_r_negg": [0.054127536714076996, 0.16489824652671814, 0.3267046809196472], "cos_innovation_negg": [0.46263542771339417, 0.5874254703521729, 0.673530101776123], "cos_apical_negg": [0.054127536714076996, 0.16489824652671814, 0.3267046809196472], "cos_Ac_negg": [0.4628351628780365, 0.5875599384307861, 0.6728527545928955], "r_norm": [1.0008635520935059, 0.99664306640625, 1.0026768445968628], "ldr": {"dL_sdil": -0.004442289471626282, "dL_bp": -0.007203981280326843, "ratio": 0.6166436722646143}}, {"step": 4600, "epoch": 9, "train_loss": 0.026353374123573303, "cos_r_negg": [0.06898064911365509, 0.17992939054965973, 0.33015739917755127], "cos_innovation_negg": [0.4801621437072754, 0.6100026965141296, 0.726508617401123], "cos_apical_negg": [0.06898064911365509, 0.17992939054965973, 0.33015739917755127], "cos_Ac_negg": [0.4800562858581543, 0.6101363897323608, 0.7259714603424072], "r_norm": [1.0048885345458984, 0.9985988140106201, 1.003064513206482]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9629, "test_loss": 0.16283142318725585}, {"step": 4700, "epoch": 10, "train_loss": 0.07447795569896698, "cos_r_negg": [0.04082931578159332, 0.13814829289913177, 0.29323896765708923], "cos_innovation_negg": [0.4813762605190277, 0.5998628735542297, 0.706299901008606], "cos_apical_negg": [0.04082931578159332, 0.13814829289913177, 0.29323896765708923], "cos_Ac_negg": [0.48167258501052856, 0.6006890535354614, 0.7043177485466003], "r_norm": [1.0003453493118286, 0.9935581088066101, 0.9974279403686523]}, {"step": 4800, "epoch": 10, "train_loss": 0.0702333077788353, "cos_r_negg": [0.06491591036319733, 0.12049730122089386, 0.2656426429748535], "cos_innovation_negg": [0.4316897690296173, 0.5156323313713074, 0.6949378848075867], "cos_apical_negg": [0.06491591036319733, 0.12049730122089386, 0.2656426429748535], "cos_Ac_negg": [0.43133971095085144, 0.5148912668228149, 0.6932864189147949], "r_norm": [0.9971155524253845, 0.9910149574279785, 0.9951145648956299]}, {"step": 4900, "epoch": 10, "train_loss": 0.07635124027729034, "cos_r_negg": [0.06220356002449989, 0.1753993034362793, 0.26844465732574463], "cos_innovation_negg": [0.47314804792404175, 0.5789248943328857, 0.7186846137046814], "cos_apical_negg": [0.06220356002449989, 0.1753993034362793, 0.26844465732574463], "cos_Ac_negg": [0.4726075530052185, 0.5764424800872803, 0.7174698114395142], "r_norm": [1.0121158361434937, 1.0073261260986328, 1.0124716758728027]}, {"step": 5000, "epoch": 10, "train_loss": 0.032667431980371475, "cos_r_negg": [0.06057484447956085, 0.13061538338661194, 0.36302196979522705], "cos_innovation_negg": [0.44369348883628845, 0.5756018161773682, 0.7298516035079956], "cos_apical_negg": [0.06057484447956085, 0.13061538338661194, 0.36302196979522705], "cos_Ac_negg": [0.44343769550323486, 0.5754837989807129, 0.7288978099822998], "r_norm": [1.0030885934829712, 0.9972087144851685, 1.0033472776412964], "ldr": {"dL_sdil": -0.004772692918777466, "dL_bp": -0.007688939571380615, "ratio": 0.6207218660609772}}, {"step": 5100, "epoch": 10, "train_loss": 0.06812694668769836, "cos_r_negg": [0.040807634592056274, 0.146819606423378, 0.3157557249069214], "cos_innovation_negg": [0.4270401895046234, 0.562659740447998, 0.6880521774291992], "cos_apical_negg": [0.040807634592056274, 0.146819606423378, 0.3157557249069214], "cos_Ac_negg": [0.42722243070602417, 0.5621775984764099, 0.6871858239173889], "r_norm": [0.9940504431724548, 0.9884777069091797, 0.991992712020874]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9637, "test_loss": 0.16795218696594238}, {"step": 5200, "epoch": 11, "train_loss": 0.005242047365754843, "cos_r_negg": [0.04723719134926796, 0.168500155210495, 0.31768786907196045], "cos_innovation_negg": [0.4486674666404724, 0.5615569353103638, 0.6777231097221375], "cos_apical_negg": [0.04723719134926796, 0.168500155210495, 0.31768786907196045], "cos_Ac_negg": [0.4490351974964142, 0.5618119835853577, 0.6759191751480103], "r_norm": [1.0135362148284912, 1.006046175956726, 1.0099883079528809]}, {"step": 5300, "epoch": 11, "train_loss": 0.04645835980772972, "cos_r_negg": [0.02789916656911373, 0.1501055508852005, 0.2888845205307007], "cos_innovation_negg": [0.39689427614212036, 0.4925505816936493, 0.6453686356544495], "cos_apical_negg": [0.02789916656911373, 0.1501055508852005, 0.2888845205307007], "cos_Ac_negg": [0.39727583527565, 0.49246904253959656, 0.6451837420463562], "r_norm": [1.0048892498016357, 0.9984314441680908, 1.0019359588623047]}, {"step": 5400, "epoch": 11, "train_loss": 0.08015645295381546, "cos_r_negg": [0.036679863929748535, 0.16330870985984802, 0.3469422459602356], "cos_innovation_negg": [0.4299176335334778, 0.5475703477859497, 0.7247656583786011], "cos_apical_negg": [0.036679863929748535, 0.16330870985984802, 0.3469422459602356], "cos_Ac_negg": [0.4303250312805176, 0.5479323863983154, 0.7244442701339722], "r_norm": [0.986892819404602, 0.9811404943466187, 0.9852945804595947]}, {"step": 5500, "epoch": 11, "train_loss": 0.10337873548269272, "cos_r_negg": [0.04536229372024536, 0.12637490034103394, 0.3078010082244873], "cos_innovation_negg": [0.4053611755371094, 0.5329481959342957, 0.7376441955566406], "cos_apical_negg": [0.04536229372024536, 0.12637490034103394, 0.3078010082244873], "cos_Ac_negg": [0.4052439332008362, 0.5332862138748169, 0.7368306517601013], "r_norm": [0.9978095293045044, 0.9923269748687744, 0.9953716993331909], "ldr": {"dL_sdil": -0.005255058407783508, "dL_bp": -0.009846597909927368, "ratio": 0.5336927998740909}}, {"step": 5600, "epoch": 11, "train_loss": 0.03063536249101162, "cos_r_negg": [0.05646223947405815, 0.16315333545207977, 0.34648674726486206], "cos_innovation_negg": [0.41272374987602234, 0.5321862101554871, 0.7480096817016602], "cos_apical_negg": [0.05646223947405815, 0.16315333545207977, 0.34648674726486206], "cos_Ac_negg": [0.4125385880470276, 0.5328807830810547, 0.7469910979270935], "r_norm": [0.9943451881408691, 0.987838625907898, 0.9920538663864136]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.964, "test_loss": 0.16722139854431153}, {"step": 5700, "epoch": 12, "train_loss": 0.01672079786658287, "cos_r_negg": [0.049960989505052567, 0.18621963262557983, 0.3282274603843689], "cos_innovation_negg": [0.41187095642089844, 0.5453330278396606, 0.7323659658432007], "cos_apical_negg": [0.049960989505052567, 0.18621963262557983, 0.3282274603843689], "cos_Ac_negg": [0.41169118881225586, 0.5457819104194641, 0.7328809499740601], "r_norm": [1.0035459995269775, 0.9972206950187683, 1.0026814937591553]}, {"step": 5800, "epoch": 12, "train_loss": 0.018970094621181488, "cos_r_negg": [0.04647641256451607, 0.14888887107372284, 0.33303457498550415], "cos_innovation_negg": [0.3900400996208191, 0.5297600626945496, 0.720571756362915], "cos_apical_negg": [0.04647641256451607, 0.14888887107372284, 0.33303457498550415], "cos_Ac_negg": [0.39014071226119995, 0.5302168130874634, 0.7207975387573242], "r_norm": [0.9923798441886902, 0.9846975803375244, 0.9881832599639893]}, {"step": 5900, "epoch": 12, "train_loss": 0.026083137840032578, "cos_r_negg": [0.04985422641038895, 0.1442699432373047, 0.31639280915260315], "cos_innovation_negg": [0.4107506275177002, 0.4937441945075989, 0.6798127889633179], "cos_apical_negg": [0.04985422641038895, 0.1442699432373047, 0.31639280915260315], "cos_Ac_negg": [0.40956953167915344, 0.49411192536354065, 0.676051914691925], "r_norm": [1.010257601737976, 1.0013244152069092, 1.0046613216400146]}, {"step": 6000, "epoch": 12, "train_loss": 0.008461544290184975, "cos_r_negg": [0.029255229979753494, 0.18780213594436646, 0.2699863910675049], "cos_innovation_negg": [0.43998152017593384, 0.5333098769187927, 0.6794975996017456], "cos_apical_negg": [0.029255229979753494, 0.18780213594436646, 0.2699863910675049], "cos_Ac_negg": [0.44005224108695984, 0.5306196808815002, 0.6777482032775879], "r_norm": [1.002807378768921, 0.9942639470100403, 0.9987096786499023], "ldr": {"dL_sdil": -0.005444705486297607, "dL_bp": -0.011139541864395142, "ratio": 0.48877283757066303}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9647, "test_loss": 0.17159284400939942}, {"step": 6100, "epoch": 13, "train_loss": 0.024622051045298576, "cos_r_negg": [0.034463364630937576, 0.15558412671089172, 0.22659939527511597], "cos_innovation_negg": [0.3993053436279297, 0.5189934968948364, 0.7001391053199768], "cos_apical_negg": [0.034463364630937576, 0.15558412671089172, 0.22659939527511597], "cos_Ac_negg": [0.39973369240760803, 0.5202004909515381, 0.7011544704437256], "r_norm": [0.9926055669784546, 0.9858974814414978, 0.9902071952819824]}, {"step": 6200, "epoch": 13, "train_loss": 0.013772055506706238, "cos_r_negg": [0.05448794364929199, 0.16238385438919067, 0.2878323197364807], "cos_innovation_negg": [0.39762502908706665, 0.5268219709396362, 0.7149753570556641], "cos_apical_negg": [0.05448794364929199, 0.16238385438919067, 0.2878323197364807], "cos_Ac_negg": [0.39821648597717285, 0.5261987447738647, 0.7132536172866821], "r_norm": [1.0000630617141724, 0.9914873242378235, 0.9952001571655273]}, {"step": 6300, "epoch": 13, "train_loss": 0.009136243723332882, "cos_r_negg": [0.054326675832271576, 0.17687208950519562, 0.30371466279029846], "cos_innovation_negg": [0.40638431906700134, 0.5090117454528809, 0.6876112222671509], "cos_apical_negg": [0.054326675832271576, 0.17687208950519562, 0.30371466279029846], "cos_Ac_negg": [0.40657466650009155, 0.5081708431243896, 0.6861210465431213], "r_norm": [1.0032062530517578, 0.9955238103866577, 0.9993560314178467]}, {"step": 6400, "epoch": 13, "train_loss": 0.01309981755912304, "cos_r_negg": [0.04377492889761925, 0.13936050236225128, 0.2471282035112381], "cos_innovation_negg": [0.40372008085250854, 0.5213161706924438, 0.6748590469360352], "cos_apical_negg": [0.04377492889761925, 0.13936050236225128, 0.2471282035112381], "cos_Ac_negg": [0.40482962131500244, 0.5230716466903687, 0.6761823892593384], "r_norm": [1.0090835094451904, 1.000382661819458, 1.0045630931854248]}, {"step": 6500, "epoch": 13, "train_loss": 0.005054917186498642, "cos_r_negg": [0.03787251189351082, 0.1691415011882782, 0.27734020352363586], "cos_innovation_negg": [0.39364761114120483, 0.4857965111732483, 0.7172523140907288], "cos_apical_negg": [0.03787251189351082, 0.1691415011882782, 0.27734020352363586], "cos_Ac_negg": [0.394325852394104, 0.48641687631607056, 0.7171387672424316], "r_norm": [1.0038480758666992, 0.9952888488769531, 0.9986175298690796], "ldr": {"dL_sdil": -0.0053085386753082275, "dL_bp": -0.009034797549247742, "ratio": 0.5875658692263922}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.965, "test_loss": 0.16982969284057617}, {"step": 6600, "epoch": 14, "train_loss": 0.01103228610008955, "cos_r_negg": [0.0637044683098793, 0.16204126179218292, 0.2569505274295807], "cos_innovation_negg": [0.4007999897003174, 0.4942467212677002, 0.7148151993751526], "cos_apical_negg": [0.0637044683098793, 0.16204126179218292, 0.2569505274295807], "cos_Ac_negg": [0.40037351846694946, 0.4896727204322815, 0.7124040126800537], "r_norm": [0.9995841383934021, 0.9915225505828857, 0.9953269362449646]}, {"step": 6700, "epoch": 14, "train_loss": 0.07044786959886551, "cos_r_negg": [0.0554986298084259, 0.1700323522090912, 0.24690739810466766], "cos_innovation_negg": [0.3683430850505829, 0.4520756006240845, 0.6897143125534058], "cos_apical_negg": [0.0554986298084259, 0.1700323522090912, 0.24690739810466766], "cos_Ac_negg": [0.3696437180042267, 0.44101646542549133, 0.6870521306991577], "r_norm": [1.0063462257385254, 0.9971489906311035, 0.9997471570968628]}, {"step": 6800, "epoch": 14, "train_loss": 0.07112330943346024, "cos_r_negg": [0.030334604904055595, 0.16273115575313568, 0.28592705726623535], "cos_innovation_negg": [0.41890501976013184, 0.5037792325019836, 0.7524046897888184], "cos_apical_negg": [0.030334604904055595, 0.16273115575313568, 0.28592705726623535], "cos_Ac_negg": [0.4216950237751007, 0.5074898600578308, 0.7542818784713745], "r_norm": [0.99994957447052, 0.992084801197052, 0.9965077638626099]}, {"step": 6900, "epoch": 14, "train_loss": 0.03472434729337692, "cos_r_negg": [0.03598427027463913, 0.19057166576385498, 0.24942214787006378], "cos_innovation_negg": [0.4031265377998352, 0.4925141930580139, 0.7411197423934937], "cos_apical_negg": [0.03598427027463913, 0.19057166576385498, 0.24942214787006378], "cos_Ac_negg": [0.4042908847332001, 0.48933911323547363, 0.7436213493347168], "r_norm": [0.9963934421539307, 0.9881312847137451, 0.990851640701294]}, {"step": 7000, "epoch": 14, "train_loss": 0.028984980657696724, "cos_r_negg": [0.045679546892642975, 0.1608377993106842, 0.24076461791992188], "cos_innovation_negg": [0.43705978989601135, 0.5236760973930359, 0.6773004531860352], "cos_apical_negg": [0.045679546892642975, 0.1608377993106842, 0.24076461791992188], "cos_Ac_negg": [0.43844664096832275, 0.5239940881729126, 0.6781637668609619], "r_norm": [1.0056698322296143, 0.9973837733268738, 1.0014251470565796], "ldr": {"dL_sdil": -0.005319669842720032, "dL_bp": -0.00823262333869934, "ratio": 0.6461694679645672}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9653, "test_loss": 0.17632402153015136}], "final": {"test_acc": 0.9653, "test_loss": 0.17632402153015136, "wall_s": 22.891586780548096, "cos_r_negg": [0.05158198997378349, 0.16588646173477173, 0.24333959817886353], "cos_innovation_negg": [0.43237197399139404, 0.5271012783050537, 0.6925674080848694], "cos_apical_negg": [0.05158198997378349, 0.16588646173477173, 0.24333959817886353], "cos_Ac_negg": [0.43461257219314575, 0.5270849466323853, 0.6901035308837891]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_raw_s2.json b/results/nuis_ctrl_v1_rho0p05_raw_s2.json new file mode 100644 index 0000000..b199532 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_raw_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_raw_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.016351228579878807, 0.03726989030838013, 0.002024024957790971], "cos_innovation_negg": [0.016769511625170708, 0.03792087361216545, 0.0024635603185743093], "cos_apical_negg": [0.016351228579878807, 0.03726989030838013, 0.002024024957790971], "cos_Ac_negg": [0.016769826412200928, 0.03792092204093933, 0.0024636252783238888], "r_norm": [4.83701229095459, 4.803536415100098, 4.886757850646973], "ldr": {"dL_sdil": -0.03889727592468262, "dL_bp": -0.2430727481842041, "ratio": 0.16002318735955415}}, {"step": 100, "epoch": 0, "train_loss": 0.26986163854599, "cos_r_negg": [0.25628721714019775, 0.24019378423690796, 0.26717162132263184], "cos_innovation_negg": [0.4942985475063324, 0.45471298694610596, 0.3989502191543579], "cos_apical_negg": [0.25628721714019775, 0.24019378423690796, 0.26717162132263184], "cos_Ac_negg": [0.49431121349334717, 0.45478397607803345, 0.3990820646286011], "r_norm": [1.5100347995758057, 1.5151798725128174, 1.522977352142334]}, {"step": 200, "epoch": 0, "train_loss": 0.2912282943725586, "cos_r_negg": [0.20758888125419617, 0.24513758718967438, 0.31017059087753296], "cos_innovation_negg": [0.5126169323921204, 0.5000236630439758, 0.41873985528945923], "cos_apical_negg": [0.20758888125419617, 0.24513758718967438, 0.31017059087753296], "cos_Ac_negg": [0.5124001502990723, 0.4993397295475006, 0.41870689392089844], "r_norm": [1.396628499031067, 1.4015787839889526, 1.4153470993041992]}, {"step": 300, "epoch": 0, "train_loss": 0.294539213180542, "cos_r_negg": [0.21960732340812683, 0.2596883177757263, 0.34202826023101807], "cos_innovation_negg": [0.5767326951026917, 0.5217605233192444, 0.5073724985122681], "cos_apical_negg": [0.21960732340812683, 0.2596883177757263, 0.34202826023101807], "cos_Ac_negg": [0.5767816305160522, 0.5216122269630432, 0.5073315501213074], "r_norm": [1.337673306465149, 1.3437931537628174, 1.356310486793518]}, {"step": 400, "epoch": 0, "train_loss": 0.14327001571655273, "cos_r_negg": [0.18934598565101624, 0.24966321885585785, 0.34589824080467224], "cos_innovation_negg": [0.5613587498664856, 0.5491464138031006, 0.5336900949478149], "cos_apical_negg": [0.18934598565101624, 0.24966321885585785, 0.34589824080467224], "cos_Ac_negg": [0.5613411664962769, 0.5487940311431885, 0.5334361791610718], "r_norm": [1.2469820976257324, 1.2484197616577148, 1.2610831260681152]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9356, "test_loss": 0.23069388809204103}, {"step": 500, "epoch": 1, "train_loss": 0.1744796633720398, "cos_r_negg": [0.14606153964996338, 0.22291582822799683, 0.2928049564361572], "cos_innovation_negg": [0.5254873037338257, 0.5276155471801758, 0.47018134593963623], "cos_apical_negg": [0.14606153964996338, 0.22291582822799683, 0.2928049564361572], "cos_Ac_negg": [0.5250939130783081, 0.5271015167236328, 0.4695916175842285], "r_norm": [1.183100938796997, 1.1890501976013184, 1.201068639755249], "ldr": {"dL_sdil": -0.007268860936164856, "dL_bp": -0.006008505821228027, "ratio": 1.2097618197329525}}, {"step": 600, "epoch": 1, "train_loss": 0.1413438618183136, "cos_r_negg": [0.16825275123119354, 0.256413072347641, 0.31469541788101196], "cos_innovation_negg": [0.5187771320343018, 0.5379265546798706, 0.5203065872192383], "cos_apical_negg": [0.16825275123119354, 0.256413072347641, 0.31469541788101196], "cos_Ac_negg": [0.518608570098877, 0.5377370715141296, 0.5199556350708008], "r_norm": [1.183385968208313, 1.1886606216430664, 1.2013578414916992]}, {"step": 700, "epoch": 1, "train_loss": 0.16877338290214539, "cos_r_negg": [0.1562815010547638, 0.2508743107318878, 0.3553983271121979], "cos_innovation_negg": [0.5167168974876404, 0.5004274249076843, 0.5420772433280945], "cos_apical_negg": [0.1562815010547638, 0.2508743107318878, 0.3553983271121979], "cos_Ac_negg": [0.5166971683502197, 0.5003420114517212, 0.5418205261230469], "r_norm": [1.1941770315170288, 1.1993513107299805, 1.2109358310699463]}, {"step": 800, "epoch": 1, "train_loss": 0.1204691007733345, "cos_r_negg": [0.1560642421245575, 0.22460466623306274, 0.3507677912712097], "cos_innovation_negg": [0.5186216831207275, 0.5664896965026855, 0.5793118476867676], "cos_apical_negg": [0.1560642421245575, 0.22460466623306274, 0.3507677912712097], "cos_Ac_negg": [0.5186062455177307, 0.5664976239204407, 0.5792034864425659], "r_norm": [1.1431457996368408, 1.1479711532592773, 1.159602403640747]}, {"step": 900, "epoch": 1, "train_loss": 0.2052237093448639, "cos_r_negg": [0.1747203916311264, 0.24243977665901184, 0.39018017053604126], "cos_innovation_negg": [0.5534815788269043, 0.6012318730354309, 0.606451690196991], "cos_apical_negg": [0.1747203916311264, 0.24243977665901184, 0.39018017053604126], "cos_Ac_negg": [0.5534197688102722, 0.601254403591156, 0.606407880783081], "r_norm": [1.1441895961761475, 1.152163028717041, 1.1619350910186768]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9491, "test_loss": 0.1914022979736328}, {"step": 1000, "epoch": 2, "train_loss": 0.026474090293049812, "cos_r_negg": [0.13155917823314667, 0.20807278156280518, 0.320778489112854], "cos_innovation_negg": [0.5283476114273071, 0.5572407245635986, 0.5860154628753662], "cos_apical_negg": [0.13155917823314667, 0.20807278156280518, 0.320778489112854], "cos_Ac_negg": [0.5282799601554871, 0.5570051670074463, 0.5858961343765259], "r_norm": [1.1425983905792236, 1.149730920791626, 1.160740852355957], "ldr": {"dL_sdil": -0.013523876667022705, "dL_bp": -0.01275336742401123, "ratio": 1.0604161408821962}}, {"step": 1100, "epoch": 2, "train_loss": 0.16789907217025757, "cos_r_negg": [0.12714336812496185, 0.19407592713832855, 0.2861878275871277], "cos_innovation_negg": [0.5129390954971313, 0.5493701696395874, 0.5685532689094543], "cos_apical_negg": [0.12714336812496185, 0.19407592713832855, 0.2861878275871277], "cos_Ac_negg": [0.5128953456878662, 0.5493298768997192, 0.5685659646987915], "r_norm": [1.1071388721466064, 1.1118988990783691, 1.1209579706192017]}, {"step": 1200, "epoch": 2, "train_loss": 0.2542726397514343, "cos_r_negg": [0.10985012352466583, 0.2414225935935974, 0.2818790674209595], "cos_innovation_negg": [0.5099470615386963, 0.5542882084846497, 0.5539080500602722], "cos_apical_negg": [0.10985012352466583, 0.2414225935935974, 0.2818790674209595], "cos_Ac_negg": [0.50983065366745, 0.5540774464607239, 0.553827702999115], "r_norm": [1.10333251953125, 1.1067118644714355, 1.1158194541931152]}, {"step": 1300, "epoch": 2, "train_loss": 0.13253039121627808, "cos_r_negg": [0.14283177256584167, 0.22870050370693207, 0.2930089831352234], "cos_innovation_negg": [0.5233677625656128, 0.5681719183921814, 0.6037907600402832], "cos_apical_negg": [0.14283177256584167, 0.22870050370693207, 0.2930089831352234], "cos_Ac_negg": [0.5233892202377319, 0.5680927634239197, 0.6038422584533691], "r_norm": [1.114945650100708, 1.1200906038284302, 1.1292599439620972]}, {"step": 1400, "epoch": 2, "train_loss": 0.16271311044692993, "cos_r_negg": [0.134611576795578, 0.20695391297340393, 0.3624567687511444], "cos_innovation_negg": [0.523209273815155, 0.5341277122497559, 0.6502496600151062], "cos_apical_negg": [0.134611576795578, 0.20695391297340393, 0.3624567687511444], "cos_Ac_negg": [0.5230029821395874, 0.5322431325912476, 0.6500287055969238], "r_norm": [1.0938799381256104, 1.0950124263763428, 1.1050587892532349]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9562, "test_loss": 0.16205402641296388}, {"step": 1500, "epoch": 3, "train_loss": 0.10747246444225311, "cos_r_negg": [0.09832113981246948, 0.20442503690719604, 0.3652954697608948], "cos_innovation_negg": [0.4885745644569397, 0.5265042781829834, 0.6296341419219971], "cos_apical_negg": [0.09832113981246948, 0.20442503690719604, 0.3652954697608948], "cos_Ac_negg": [0.48862993717193604, 0.5263822078704834, 0.6293399333953857], "r_norm": [1.0852539539337158, 1.0890161991119385, 1.0972671508789062], "ldr": {"dL_sdil": -0.005270898342132568, "dL_bp": -0.005416393280029297, "ratio": 0.9731380403204507}}, {"step": 1600, "epoch": 3, "train_loss": 0.1145443543791771, "cos_r_negg": [0.08535800874233246, 0.21306142210960388, 0.2815054953098297], "cos_innovation_negg": [0.5080737471580505, 0.5409162044525146, 0.6195935606956482], "cos_apical_negg": [0.08535800874233246, 0.21306142210960388, 0.2815054953098297], "cos_Ac_negg": [0.5081391334533691, 0.5405891537666321, 0.6192511320114136], "r_norm": [1.0662376880645752, 1.0716567039489746, 1.0792346000671387]}, {"step": 1700, "epoch": 3, "train_loss": 0.11719862371683121, "cos_r_negg": [0.09216548502445221, 0.18955466151237488, 0.31066644191741943], "cos_innovation_negg": [0.49031540751457214, 0.5224769711494446, 0.6692118644714355], "cos_apical_negg": [0.09216548502445221, 0.18955466151237488, 0.31066644191741943], "cos_Ac_negg": [0.4902868866920471, 0.521754264831543, 0.669191837310791], "r_norm": [1.0857101678848267, 1.0888488292694092, 1.098669171333313]}, {"step": 1800, "epoch": 3, "train_loss": 0.01523064449429512, "cos_r_negg": [0.10776516795158386, 0.2166716456413269, 0.32407575845718384], "cos_innovation_negg": [0.5026174783706665, 0.5843934416770935, 0.6322363615036011], "cos_apical_negg": [0.10776516795158386, 0.2166716456413269, 0.32407575845718384], "cos_Ac_negg": [0.5025508403778076, 0.584214448928833, 0.6320405602455139], "r_norm": [1.0647015571594238, 1.067875623703003, 1.0777519941329956]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9593, "test_loss": 0.15696399307250977}, {"step": 1900, "epoch": 4, "train_loss": 0.04739385098218918, "cos_r_negg": [0.12036898732185364, 0.22712752223014832, 0.30405524373054504], "cos_innovation_negg": [0.46392714977264404, 0.5529402494430542, 0.616125226020813], "cos_apical_negg": [0.12036898732185364, 0.22712752223014832, 0.30405524373054504], "cos_Ac_negg": [0.46389687061309814, 0.5529607534408569, 0.6161174178123474], "r_norm": [1.0681471824645996, 1.0699501037597656, 1.0794968605041504]}, {"step": 2000, "epoch": 4, "train_loss": 0.19351817667484283, "cos_r_negg": [0.08464972674846649, 0.21727022528648376, 0.31119439005851746], "cos_innovation_negg": [0.45414140820503235, 0.5203954577445984, 0.633937656879425], "cos_apical_negg": [0.08464972674846649, 0.21727022528648376, 0.31119439005851746], "cos_Ac_negg": [0.45406073331832886, 0.5195996761322021, 0.6337117552757263], "r_norm": [1.0569688081741333, 1.0599610805511475, 1.0669286251068115], "ldr": {"dL_sdil": -0.005297437310218811, "dL_bp": -0.006804227828979492, "ratio": 0.7785508427064718}}, {"step": 2100, "epoch": 4, "train_loss": 0.04885556176304817, "cos_r_negg": [0.10121302306652069, 0.19666436314582825, 0.30896008014678955], "cos_innovation_negg": [0.4326215088367462, 0.4832768738269806, 0.5959169864654541], "cos_apical_negg": [0.10121302306652069, 0.19666436314582825, 0.30896008014678955], "cos_Ac_negg": [0.4325896203517914, 0.4830506145954132, 0.595726728439331], "r_norm": [1.064927577972412, 1.0686793327331543, 1.079019546508789]}, {"step": 2200, "epoch": 4, "train_loss": 0.04723121225833893, "cos_r_negg": [0.07624703645706177, 0.1619018018245697, 0.34119218587875366], "cos_innovation_negg": [0.442100465297699, 0.5139723420143127, 0.6453237533569336], "cos_apical_negg": [0.07624703645706177, 0.1619018018245697, 0.34119218587875366], "cos_Ac_negg": [0.4422610402107239, 0.5141171216964722, 0.6447874903678894], "r_norm": [1.0270897150039673, 1.0269055366516113, 1.0347896814346313]}, {"step": 2300, "epoch": 4, "train_loss": 0.2684209942817688, "cos_r_negg": [0.08285887539386749, 0.19121122360229492, 0.32221493124961853], "cos_innovation_negg": [0.45247018337249756, 0.48010221123695374, 0.6487439870834351], "cos_apical_negg": [0.08285887539386749, 0.19121122360229492, 0.32221493124961853], "cos_Ac_negg": [0.4524266719818115, 0.47961270809173584, 0.6485801935195923], "r_norm": [1.0334091186523438, 1.037372350692749, 1.0438393354415894]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9602, "test_loss": 0.16592668800354005}, {"step": 2400, "epoch": 5, "train_loss": 0.13917739689350128, "cos_r_negg": [0.07603222131729126, 0.20001882314682007, 0.31610608100891113], "cos_innovation_negg": [0.43403303623199463, 0.5018294453620911, 0.6676158905029297], "cos_apical_negg": [0.07603222131729126, 0.20001882314682007, 0.31610608100891113], "cos_Ac_negg": [0.43397489190101624, 0.5016034841537476, 0.6676636934280396], "r_norm": [1.0435519218444824, 1.046731948852539, 1.0527172088623047]}, {"step": 2500, "epoch": 5, "train_loss": 0.03787464275956154, "cos_r_negg": [0.06920893490314484, 0.16124743223190308, 0.1818993240594864], "cos_innovation_negg": [0.41523709893226624, 0.49587690830230713, 0.6368871927261353], "cos_apical_negg": [0.06920893490314484, 0.16124743223190308, 0.1818993240594864], "cos_Ac_negg": [0.41509315371513367, 0.4955156147480011, 0.6368773579597473], "r_norm": [1.0376766920089722, 1.0383743047714233, 1.0456738471984863], "ldr": {"dL_sdil": -0.007077902555465698, "dL_bp": -0.008019566535949707, "ratio": 0.8825791922465179}}, {"step": 2600, "epoch": 5, "train_loss": 0.11516404896974564, "cos_r_negg": [0.07850246131420135, 0.19498541951179504, 0.25892364978790283], "cos_innovation_negg": [0.42178723216056824, 0.49689653515815735, 0.6748406887054443], "cos_apical_negg": [0.07850246131420135, 0.19498541951179504, 0.25892364978790283], "cos_Ac_negg": [0.42181849479675293, 0.49699676036834717, 0.6747892498970032], "r_norm": [1.0432583093643188, 1.0430543422698975, 1.049689769744873]}, {"step": 2700, "epoch": 5, "train_loss": 0.06027599424123764, "cos_r_negg": [0.08669734001159668, 0.17684853076934814, 0.2351863980293274], "cos_innovation_negg": [0.42396074533462524, 0.5160149931907654, 0.6822075843811035], "cos_apical_negg": [0.08669734001159668, 0.17684853076934814, 0.2351863980293274], "cos_Ac_negg": [0.42389774322509766, 0.5159808993339539, 0.6821030378341675], "r_norm": [1.0221501588821411, 1.0236858129501343, 1.0319340229034424]}, {"step": 2800, "epoch": 5, "train_loss": 0.1499190777540207, "cos_r_negg": [0.04576827958226204, 0.17267780005931854, 0.31112733483314514], "cos_innovation_negg": [0.42414432764053345, 0.5048084855079651, 0.697646975517273], "cos_apical_negg": [0.04576827958226204, 0.17267780005931854, 0.31112733483314514], "cos_Ac_negg": [0.4240718483924866, 0.5047957897186279, 0.6973773241043091], "r_norm": [1.04861581325531, 1.0489072799682617, 1.0542129278182983]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9629, "test_loss": 0.1548251091003418}, {"step": 2900, "epoch": 6, "train_loss": 0.035604722797870636, "cos_r_negg": [0.05956468731164932, 0.16987286508083344, 0.26520663499832153], "cos_innovation_negg": [0.42340803146362305, 0.5266742706298828, 0.6552305817604065], "cos_apical_negg": [0.05956468731164932, 0.16987286508083344, 0.26520663499832153], "cos_Ac_negg": [0.4233049750328064, 0.5263910889625549, 0.6550119519233704], "r_norm": [1.0403480529785156, 1.0421652793884277, 1.0484814643859863]}, {"step": 3000, "epoch": 6, "train_loss": 0.3359282910823822, "cos_r_negg": [0.04479443281888962, 0.16483263671398163, 0.2805774509906769], "cos_innovation_negg": [0.4326283633708954, 0.5074151754379272, 0.708055853843689], "cos_apical_negg": [0.04479443281888962, 0.16483263671398163, 0.2805774509906769], "cos_Ac_negg": [0.4326822757720947, 0.5071079134941101, 0.7078747749328613], "r_norm": [1.041213035583496, 1.041967749595642, 1.0465790033340454], "ldr": {"dL_sdil": -0.007627695798873901, "dL_bp": -0.009850084781646729, "ratio": 0.7743786950023297}}, {"step": 3100, "epoch": 6, "train_loss": 0.10645271837711334, "cos_r_negg": [0.06096420809626579, 0.1788126826286316, 0.3558082580566406], "cos_innovation_negg": [0.4160324037075043, 0.5564993023872375, 0.7117502093315125], "cos_apical_negg": [0.06096420809626579, 0.1788126826286316, 0.3558082580566406], "cos_Ac_negg": [0.4161112308502197, 0.5564723014831543, 0.7115498781204224], "r_norm": [1.0394959449768066, 1.0398221015930176, 1.0456818342208862]}, {"step": 3200, "epoch": 6, "train_loss": 0.11745772510766983, "cos_r_negg": [0.0737219750881195, 0.11274389922618866, 0.33926478028297424], "cos_innovation_negg": [0.4151924252510071, 0.47463738918304443, 0.6780823469161987], "cos_apical_negg": [0.0737219750881195, 0.11274389922618866, 0.33926478028297424], "cos_Ac_negg": [0.41510018706321716, 0.4741966724395752, 0.6766713857650757], "r_norm": [1.0113651752471924, 1.0123536586761475, 1.017418384552002]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9623, "test_loss": 0.15054265441894532}, {"step": 3300, "epoch": 7, "train_loss": 0.07802364975214005, "cos_r_negg": [0.06007787585258484, 0.1528978794813156, 0.26827648282051086], "cos_innovation_negg": [0.3758278787136078, 0.48382699489593506, 0.6397097110748291], "cos_apical_negg": [0.06007787585258484, 0.1528978794813156, 0.26827648282051086], "cos_Ac_negg": [0.3757668733596802, 0.4831589162349701, 0.6392779350280762], "r_norm": [1.0247358083724976, 1.024651288986206, 1.0298504829406738]}, {"step": 3400, "epoch": 7, "train_loss": 0.18196818232536316, "cos_r_negg": [0.06407380104064941, 0.16094107925891876, 0.3179914057254791], "cos_innovation_negg": [0.3928675949573517, 0.5216501951217651, 0.6770020127296448], "cos_apical_negg": [0.06407380104064941, 0.16094107925891876, 0.3179914057254791], "cos_Ac_negg": [0.39304113388061523, 0.5214177370071411, 0.6761513948440552], "r_norm": [1.0437992811203003, 1.0431475639343262, 1.0476539134979248]}, {"step": 3500, "epoch": 7, "train_loss": 0.01314765214920044, "cos_r_negg": [0.05806967616081238, 0.120930515229702, 0.2554439306259155], "cos_innovation_negg": [0.3682251572608948, 0.489846408367157, 0.6930050849914551], "cos_apical_negg": [0.05806967616081238, 0.120930515229702, 0.2554439306259155], "cos_Ac_negg": [0.3684997260570526, 0.4897700548171997, 0.6925106644630432], "r_norm": [1.0158754587173462, 1.0165367126464844, 1.0194305181503296], "ldr": {"dL_sdil": -0.004324793815612793, "dL_bp": -0.0059500932693481445, "ratio": 0.7268447097950433}}, {"step": 3600, "epoch": 7, "train_loss": 0.01800311543047428, "cos_r_negg": [0.04473032429814339, 0.15140068531036377, 0.30699431896209717], "cos_innovation_negg": [0.3747236430644989, 0.5349630117416382, 0.6829509735107422], "cos_apical_negg": [0.04473032429814339, 0.15140068531036377, 0.30699431896209717], "cos_Ac_negg": [0.3746451735496521, 0.534785807132721, 0.6827178001403809], "r_norm": [1.0251399278640747, 1.025629997253418, 1.0305159091949463]}, {"step": 3700, "epoch": 7, "train_loss": 0.0316157303750515, "cos_r_negg": [0.0652296394109726, 0.1455095410346985, 0.26867449283599854], "cos_innovation_negg": [0.39038708806037903, 0.5224292278289795, 0.6376640796661377], "cos_apical_negg": [0.0652296394109726, 0.1455095410346985, 0.26867449283599854], "cos_Ac_negg": [0.39025378227233887, 0.5222373604774475, 0.6374752521514893], "r_norm": [1.0288434028625488, 1.028891921043396, 1.0335458517074585]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9622, "test_loss": 0.16071832809448242}, {"step": 3800, "epoch": 8, "train_loss": 0.06945928931236267, "cos_r_negg": [0.033895138651132584, 0.14156129956245422, 0.24404315650463104], "cos_innovation_negg": [0.379380464553833, 0.463906854391098, 0.6076244711875916], "cos_apical_negg": [0.033895138651132584, 0.14156129956245422, 0.24404315650463104], "cos_Ac_negg": [0.37934327125549316, 0.4638002812862396, 0.6072016954421997], "r_norm": [1.0284082889556885, 1.0292755365371704, 1.0341817140579224]}, {"step": 3900, "epoch": 8, "train_loss": 0.14554038643836975, "cos_r_negg": [0.027081657201051712, 0.11834404617547989, 0.206547349691391], "cos_innovation_negg": [0.37685123085975647, 0.4845139980316162, 0.6323327422142029], "cos_apical_negg": [0.027081657201051712, 0.11834404617547989, 0.206547349691391], "cos_Ac_negg": [0.3768630623817444, 0.4843243956565857, 0.6320868730545044], "r_norm": [1.0301066637039185, 1.0308501720428467, 1.035128116607666]}, {"step": 4000, "epoch": 8, "train_loss": 0.04828101396560669, "cos_r_negg": [0.057887621223926544, 0.12340007722377777, 0.2330154925584793], "cos_innovation_negg": [0.3884826898574829, 0.4943521022796631, 0.6579826474189758], "cos_apical_negg": [0.057887621223926544, 0.12340007722377777, 0.2330154925584793], "cos_Ac_negg": [0.3882058262825012, 0.49406445026397705, 0.6578570604324341], "r_norm": [1.0386191606521606, 1.0391212701797485, 1.0441747903823853], "ldr": {"dL_sdil": -0.006395861506462097, "dL_bp": -0.009474202990531921, "ratio": 0.6750817470191238}}, {"step": 4100, "epoch": 8, "train_loss": 0.01813218742609024, "cos_r_negg": [0.03977227583527565, 0.09697447717189789, 0.28262269496917725], "cos_innovation_negg": [0.3641955256462097, 0.5063440799713135, 0.6412769556045532], "cos_apical_negg": [0.03977227583527565, 0.09697447717189789, 0.28262269496917725], "cos_Ac_negg": [0.3640381395816803, 0.5061465501785278, 0.6398627758026123], "r_norm": [1.000592827796936, 0.9990374445915222, 1.0047214031219482]}, {"step": 4200, "epoch": 8, "train_loss": 0.10983727872371674, "cos_r_negg": [0.050268735736608505, 0.11894770711660385, 0.20759624242782593], "cos_innovation_negg": [0.33492615818977356, 0.489298015832901, 0.6386975049972534], "cos_apical_negg": [0.050268735736608505, 0.11894770711660385, 0.20759624242782593], "cos_Ac_negg": [0.33475756645202637, 0.4889370799064636, 0.6381212472915649], "r_norm": [1.0326571464538574, 1.0303642749786377, 1.0351231098175049]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9621, "test_loss": 0.16804282608032226}, {"step": 4300, "epoch": 9, "train_loss": 0.03169582039117813, "cos_r_negg": [0.056716836988925934, 0.11747080087661743, 0.3277408480644226], "cos_innovation_negg": [0.37457406520843506, 0.5029706954956055, 0.6850324869155884], "cos_apical_negg": [0.056716836988925934, 0.11747080087661743, 0.3277408480644226], "cos_Ac_negg": [0.37432217597961426, 0.5027915239334106, 0.6845425367355347], "r_norm": [1.0123813152313232, 1.011573076248169, 1.0154955387115479]}, {"step": 4400, "epoch": 9, "train_loss": 0.027957648038864136, "cos_r_negg": [0.030482858419418335, 0.12035266309976578, 0.2948065996170044], "cos_innovation_negg": [0.32133054733276367, 0.4465826153755188, 0.6455540060997009], "cos_apical_negg": [0.030482858419418335, 0.12035266309976578, 0.2948065996170044], "cos_Ac_negg": [0.32133758068084717, 0.4465687870979309, 0.645000159740448], "r_norm": [1.0165274143218994, 1.016871452331543, 1.020594835281372]}, {"step": 4500, "epoch": 9, "train_loss": 0.11018411070108414, "cos_r_negg": [0.06123633682727814, 0.08193018287420273, 0.29613497853279114], "cos_innovation_negg": [0.3565700352191925, 0.4443163275718689, 0.6262005567550659], "cos_apical_negg": [0.06123633682727814, 0.08193018287420273, 0.29613497853279114], "cos_Ac_negg": [0.3562677502632141, 0.44396141171455383, 0.6249265670776367], "r_norm": [1.030700922012329, 1.0280333757400513, 1.0313823223114014], "ldr": {"dL_sdil": -0.00478971004486084, "dL_bp": -0.008446455001831055, "ratio": 0.5670674908967737}}, {"step": 4600, "epoch": 9, "train_loss": 0.01623917929828167, "cos_r_negg": [0.03702297806739807, 0.11786529421806335, 0.2312566190958023], "cos_innovation_negg": [0.3388504385948181, 0.4537273049354553, 0.6113945245742798], "cos_apical_negg": [0.03702297806739807, 0.11786529421806335, 0.2312566190958023], "cos_Ac_negg": [0.33872902393341064, 0.45364511013031006, 0.611278772354126], "r_norm": [1.0481256246566772, 1.047587275505066, 1.053357720375061]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9631, "test_loss": 0.1681679542541504}, {"step": 4700, "epoch": 10, "train_loss": 0.04324685037136078, "cos_r_negg": [0.032492224127054214, 0.10170533508062363, 0.2886511981487274], "cos_innovation_negg": [0.35580337047576904, 0.4843495488166809, 0.633918285369873], "cos_apical_negg": [0.032492224127054214, 0.10170533508062363, 0.2886511981487274], "cos_Ac_negg": [0.3555873930454254, 0.484294593334198, 0.6337323188781738], "r_norm": [1.0124523639678955, 1.0106303691864014, 1.0152283906936646]}, {"step": 4800, "epoch": 10, "train_loss": 0.03915926069021225, "cos_r_negg": [0.035827070474624634, 0.08516813814640045, 0.2266768515110016], "cos_innovation_negg": [0.3622944951057434, 0.47491222620010376, 0.6855206489562988], "cos_apical_negg": [0.035827070474624634, 0.08516813814640045, 0.2266768515110016], "cos_Ac_negg": [0.36222898960113525, 0.4747999906539917, 0.6850283145904541], "r_norm": [1.0250439643859863, 1.0226080417633057, 1.0261266231536865]}, {"step": 4900, "epoch": 10, "train_loss": 0.03003416396677494, "cos_r_negg": [0.04176147282123566, 0.0818956270813942, 0.29116857051849365], "cos_innovation_negg": [0.3360501527786255, 0.48425179719924927, 0.6524425745010376], "cos_apical_negg": [0.04176147282123566, 0.0818956270813942, 0.29116857051849365], "cos_Ac_negg": [0.3366638123989105, 0.48598137497901917, 0.6509617567062378], "r_norm": [1.040522813796997, 1.038049578666687, 1.0413061380386353]}, {"step": 5000, "epoch": 10, "train_loss": 0.00699983723461628, "cos_r_negg": [0.04777536913752556, 0.10052882134914398, 0.35366737842559814], "cos_innovation_negg": [0.39586955308914185, 0.5053967833518982, 0.7033608555793762], "cos_apical_negg": [0.04777536913752556, 0.10052882134914398, 0.35366737842559814], "cos_Ac_negg": [0.39542049169540405, 0.5055422186851501, 0.7029327154159546], "r_norm": [1.0290982723236084, 1.028825044631958, 1.032777190208435], "ldr": {"dL_sdil": -0.0054297298192977905, "dL_bp": -0.0091448575258255, "ratio": 0.5937467920045755}}, {"step": 5100, "epoch": 10, "train_loss": 0.08448980003595352, "cos_r_negg": [0.028528505936264992, 0.11624658107757568, 0.23372116684913635], "cos_innovation_negg": [0.3551589846611023, 0.45044177770614624, 0.6218096017837524], "cos_apical_negg": [0.028528505936264992, 0.11624658107757568, 0.23372116684913635], "cos_Ac_negg": [0.3551538586616516, 0.45022547245025635, 0.6192550659179688], "r_norm": [1.0273935794830322, 1.0254517793655396, 1.0301222801208496]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.964, "test_loss": 0.1677323959350586}, {"step": 5200, "epoch": 11, "train_loss": 0.050172507762908936, "cos_r_negg": [0.03189491480588913, 0.06905107200145721, 0.22596481442451477], "cos_innovation_negg": [0.3258989453315735, 0.41669976711273193, 0.5764374136924744], "cos_apical_negg": [0.03189491480588913, 0.06905107200145721, 0.22596481442451477], "cos_Ac_negg": [0.32583022117614746, 0.41702890396118164, 0.5754662752151489], "r_norm": [1.0290672779083252, 1.0272388458251953, 1.0313624143600464]}, {"step": 5300, "epoch": 11, "train_loss": 0.022706232964992523, "cos_r_negg": [0.007912812754511833, 0.10974112153053284, 0.2908024489879608], "cos_innovation_negg": [0.3333604335784912, 0.47096559405326843, 0.6317276358604431], "cos_apical_negg": [0.007912812754511833, 0.10974112153053284, 0.2908024489879608], "cos_Ac_negg": [0.33340656757354736, 0.47087275981903076, 0.6308565735816956], "r_norm": [1.0167818069458008, 1.015000343322754, 1.0184882879257202]}, {"step": 5400, "epoch": 11, "train_loss": 0.069577157497406, "cos_r_negg": [0.02372204139828682, 0.10080938041210175, 0.3069659471511841], "cos_innovation_negg": [0.35026657581329346, 0.49211204051971436, 0.6689143180847168], "cos_apical_negg": [0.02372204139828682, 0.10080938041210175, 0.3069659471511841], "cos_Ac_negg": [0.3503696918487549, 0.4910362958908081, 0.6607381105422974], "r_norm": [1.0235652923583984, 1.02238130569458, 1.0265272855758667]}, {"step": 5500, "epoch": 11, "train_loss": 0.08107070624828339, "cos_r_negg": [0.06123141944408417, 0.06525152921676636, 0.2569100558757782], "cos_innovation_negg": [0.3118323087692261, 0.43399563431739807, 0.619516134262085], "cos_apical_negg": [0.06123141944408417, 0.06525152921676636, 0.2569100558757782], "cos_Ac_negg": [0.3117569088935852, 0.43471184372901917, 0.6182646751403809], "r_norm": [1.0474112033843994, 1.0459380149841309, 1.0505635738372803], "ldr": {"dL_sdil": -0.006547495722770691, "dL_bp": -0.011910423636436462, "ratio": 0.549728197974465}}, {"step": 5600, "epoch": 11, "train_loss": 0.011818649247288704, "cos_r_negg": [0.04048186540603638, 0.0688992589712143, 0.26414743065834045], "cos_innovation_negg": [0.3299340605735779, 0.4390382766723633, 0.6339954137802124], "cos_apical_negg": [0.04048186540603638, 0.0688992589712143, 0.26414743065834045], "cos_Ac_negg": [0.3301112651824951, 0.43929266929626465, 0.6339505314826965], "r_norm": [1.0357422828674316, 1.034662127494812, 1.0386290550231934]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9627, "test_loss": 0.17148425521850585}, {"step": 5700, "epoch": 12, "train_loss": 0.013509337790310383, "cos_r_negg": [0.028665881603956223, 0.09053084254264832, 0.2626854181289673], "cos_innovation_negg": [0.35355842113494873, 0.45541912317276, 0.6987771987915039], "cos_apical_negg": [0.028665881603956223, 0.09053084254264832, 0.2626854181289673], "cos_Ac_negg": [0.3536085784435272, 0.45505645871162415, 0.6985242366790771], "r_norm": [1.0178207159042358, 1.0156784057617188, 1.0203258991241455]}, {"step": 5800, "epoch": 12, "train_loss": 0.003549558110535145, "cos_r_negg": [0.04264039546251297, 0.0927126333117485, 0.28832879662513733], "cos_innovation_negg": [0.3411066234111786, 0.4813883900642395, 0.6470040082931519], "cos_apical_negg": [0.04264039546251297, 0.0927126333117485, 0.28832879662513733], "cos_Ac_negg": [0.3416808247566223, 0.4815107583999634, 0.64670330286026], "r_norm": [1.0133148431777954, 1.0111066102981567, 1.0143961906433105]}, {"step": 5900, "epoch": 12, "train_loss": 0.029704447835683823, "cos_r_negg": [0.03861469775438309, 0.08688181638717651, 0.3242892324924469], "cos_innovation_negg": [0.3356017470359802, 0.4597376585006714, 0.6622301340103149], "cos_apical_negg": [0.03861469775438309, 0.08688181638717651, 0.3242892324924469], "cos_Ac_negg": [0.3360610604286194, 0.46034476161003113, 0.6626713275909424], "r_norm": [1.0380618572235107, 1.036217451095581, 1.0400032997131348]}, {"step": 6000, "epoch": 12, "train_loss": 0.008316019549965858, "cos_r_negg": [0.07833492010831833, 0.07370015978813171, 0.2945588231086731], "cos_innovation_negg": [0.33993470668792725, 0.45780181884765625, 0.6740881204605103], "cos_apical_negg": [0.07833492010831833, 0.07370015978813171, 0.2945588231086731], "cos_Ac_negg": [0.33993780612945557, 0.45785602927207947, 0.6738462448120117], "r_norm": [1.0219439268112183, 1.020371675491333, 1.02308189868927], "ldr": {"dL_sdil": -0.006641268730163574, "dL_bp": -0.013737350702285767, "ratio": 0.48344610792083287}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9639, "test_loss": 0.17087281951904296}, {"step": 6100, "epoch": 13, "train_loss": 0.024254435673356056, "cos_r_negg": [0.054426658898591995, 0.06026718392968178, 0.30080968141555786], "cos_innovation_negg": [0.363950252532959, 0.468508243560791, 0.6378719806671143], "cos_apical_negg": [0.054426658898591995, 0.06026718392968178, 0.30080968141555786], "cos_Ac_negg": [0.3638584017753601, 0.4686540961265564, 0.6325234174728394], "r_norm": [1.0193496942520142, 1.0158413648605347, 1.0197868347167969]}, {"step": 6200, "epoch": 13, "train_loss": 0.007111395709216595, "cos_r_negg": [0.047527872025966644, 0.06168128550052643, 0.29947179555892944], "cos_innovation_negg": [0.3592364192008972, 0.48100411891937256, 0.6827964782714844], "cos_apical_negg": [0.047527872025966644, 0.06168128550052643, 0.29947179555892944], "cos_Ac_negg": [0.35922491550445557, 0.4812743663787842, 0.6823273301124573], "r_norm": [1.0195106267929077, 1.0165306329727173, 1.0211217403411865]}, {"step": 6300, "epoch": 13, "train_loss": 0.022502919659018517, "cos_r_negg": [0.037784136831760406, 0.0038733510300517082, 0.28506940603256226], "cos_innovation_negg": [0.31635022163391113, 0.45350325107574463, 0.6665425300598145], "cos_apical_negg": [0.037784136831760406, 0.0038733510300517082, 0.28506940603256226], "cos_Ac_negg": [0.31751328706741333, 0.4573613703250885, 0.666191816329956], "r_norm": [1.0003410577774048, 0.9972989559173584, 1.0002323389053345]}, {"step": 6400, "epoch": 13, "train_loss": 0.05799753591418266, "cos_r_negg": [0.06157517433166504, 0.07098328322172165, 0.2955743670463562], "cos_innovation_negg": [0.342886745929718, 0.4874573349952698, 0.7034364342689514], "cos_apical_negg": [0.06157517433166504, 0.07098328322172165, 0.2955743670463562], "cos_Ac_negg": [0.34280282258987427, 0.4878469407558441, 0.7031430006027222], "r_norm": [1.0177587270736694, 1.0156582593917847, 1.019577980041504]}, {"step": 6500, "epoch": 13, "train_loss": 0.0017245321068912745, "cos_r_negg": [0.04962688684463501, 0.09194190055131912, 0.2958463728427887], "cos_innovation_negg": [0.34599483013153076, 0.47345688939094543, 0.7042845487594604], "cos_apical_negg": [0.04962688684463501, 0.09194190055131912, 0.2958463728427887], "cos_Ac_negg": [0.34579819440841675, 0.47357726097106934, 0.703488826751709], "r_norm": [1.019076943397522, 1.0159770250320435, 1.0191694498062134], "ldr": {"dL_sdil": -0.004712209105491638, "dL_bp": -0.009029418230056763, "ratio": 0.5218729474514567}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9635, "test_loss": 0.17431141624450683}, {"step": 6600, "epoch": 14, "train_loss": 0.007117562927305698, "cos_r_negg": [0.04811757057905197, 0.0898047387599945, 0.3075282573699951], "cos_innovation_negg": [0.3429800868034363, 0.450987845659256, 0.7204975485801697], "cos_apical_negg": [0.04811757057905197, 0.0898047387599945, 0.3075282573699951], "cos_Ac_negg": [0.34369322657585144, 0.4519035518169403, 0.7196357250213623], "r_norm": [1.021923303604126, 1.019662857055664, 1.0226385593414307]}, {"step": 6700, "epoch": 14, "train_loss": 0.07540994137525558, "cos_r_negg": [0.032378241419792175, 0.07785306870937347, 0.2887074947357178], "cos_innovation_negg": [0.3094223141670227, 0.4115906357765198, 0.7188887000083923], "cos_apical_negg": [0.032378241419792175, 0.07785306870937347, 0.2887074947357178], "cos_Ac_negg": [0.3104996085166931, 0.4201647639274597, 0.7180632948875427], "r_norm": [1.0125173330307007, 1.0098356008529663, 1.0116264820098877]}, {"step": 6800, "epoch": 14, "train_loss": 0.04273519665002823, "cos_r_negg": [0.05510205775499344, 0.06265251338481903, 0.26751765608787537], "cos_innovation_negg": [0.3215644657611847, 0.427243709564209, 0.6946721076965332], "cos_apical_negg": [0.05510205775499344, 0.06265251338481903, 0.26751765608787537], "cos_Ac_negg": [0.32179802656173706, 0.4275987446308136, 0.6918890476226807], "r_norm": [1.0124951601028442, 1.010005235671997, 1.0128991603851318]}, {"step": 6900, "epoch": 14, "train_loss": 0.020757019519805908, "cos_r_negg": [0.043970074504613876, 0.03675880283117294, 0.2528495788574219], "cos_innovation_negg": [0.337636798620224, 0.45160892605781555, 0.7302550077438354], "cos_apical_negg": [0.043970074504613876, 0.03675880283117294, 0.2528495788574219], "cos_Ac_negg": [0.3385072946548462, 0.4523769021034241, 0.726183295249939], "r_norm": [1.0186567306518555, 1.0157909393310547, 1.0181643962860107]}, {"step": 7000, "epoch": 14, "train_loss": 0.02624410018324852, "cos_r_negg": [0.053022582083940506, 0.06284834444522858, 0.2681516408920288], "cos_innovation_negg": [0.3493904769420624, 0.4494837522506714, 0.6895380020141602], "cos_apical_negg": [0.053022582083940506, 0.06284834444522858, 0.2681516408920288], "cos_Ac_negg": [0.34938371181488037, 0.44941359758377075, 0.6903500556945801], "r_norm": [1.025800108909607, 1.023171067237854, 1.0263149738311768], "ldr": {"dL_sdil": -0.004941582679748535, "dL_bp": -0.008859783411026001, "ratio": 0.5577543434751164}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9634, "test_loss": 0.18059595985412596}], "final": {"test_acc": 0.9634, "test_loss": 0.18059595985412596, "wall_s": 28.10284924507141, "cos_r_negg": [0.035696741193532944, 0.08156191557645798, 0.3231891691684723], "cos_innovation_negg": [0.3569589853286743, 0.4908323287963867, 0.7206974625587463], "cos_apical_negg": [0.035696741193532944, 0.08156191557645798, 0.3231891691684723], "cos_Ac_negg": [0.35695669054985046, 0.4909154772758484, 0.7197968363761902]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_raw_s3.json b/results/nuis_ctrl_v1_rho0p05_raw_s3.json new file mode 100644 index 0000000..21f4eee --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_raw_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_raw_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.015642719343304634, 0.013174558989703655, 0.005119418725371361], "cos_innovation_negg": [-0.015008550137281418, 0.013928058557212353, 0.006005333736538887], "cos_apical_negg": [-0.015642719343304634, 0.013174558989703655, 0.005119418725371361], "cos_Ac_negg": [-0.015008732676506042, 0.013928016647696495, 0.006005217786878347], "r_norm": [4.937504768371582, 4.780068397521973, 4.870107173919678], "ldr": {"dL_sdil": -0.00045490264892578125, "dL_bp": -0.22743654251098633, "ratio": 0.0020001299874834633}}, {"step": 100, "epoch": 0, "train_loss": 0.2647920846939087, "cos_r_negg": [0.23171938955783844, 0.23704123497009277, 0.23391561210155487], "cos_innovation_negg": [0.4368886947631836, 0.41796720027923584, 0.3002517819404602], "cos_apical_negg": [0.23171938955783844, 0.23704123497009277, 0.23391561210155487], "cos_Ac_negg": [0.4369359016418457, 0.4181079864501953, 0.30021610856056213], "r_norm": [1.5268943309783936, 1.5047580003738403, 1.513484239578247]}, {"step": 200, "epoch": 0, "train_loss": 0.26878318190574646, "cos_r_negg": [0.2212350070476532, 0.2375158816576004, 0.24544546008110046], "cos_innovation_negg": [0.4542025625705719, 0.4362070560455322, 0.29868799448013306], "cos_apical_negg": [0.2212350070476532, 0.2375158816576004, 0.24544546008110046], "cos_Ac_negg": [0.4543095529079437, 0.43619081377983093, 0.29826998710632324], "r_norm": [1.3969652652740479, 1.3823130130767822, 1.3911058902740479]}, {"step": 300, "epoch": 0, "train_loss": 0.23749800026416779, "cos_r_negg": [0.20038194954395294, 0.25258520245552063, 0.27908068895339966], "cos_innovation_negg": [0.4624326825141907, 0.45986998081207275, 0.4063308835029602], "cos_apical_negg": [0.20038194954395294, 0.25258520245552063, 0.27908068895339966], "cos_Ac_negg": [0.46241092681884766, 0.45987874269485474, 0.4063032865524292], "r_norm": [1.369203805923462, 1.355884313583374, 1.3696670532226562]}, {"step": 400, "epoch": 0, "train_loss": 0.2108488380908966, "cos_r_negg": [0.1951042264699936, 0.2563267648220062, 0.2589138150215149], "cos_innovation_negg": [0.5284556150436401, 0.5371371507644653, 0.41852307319641113], "cos_apical_negg": [0.1951042264699936, 0.2563267648220062, 0.2589138150215149], "cos_Ac_negg": [0.5284525156021118, 0.5371414422988892, 0.41846397519111633], "r_norm": [1.2580506801605225, 1.251002311706543, 1.2639069557189941]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9421, "test_loss": 0.20638164520263672}, {"step": 500, "epoch": 1, "train_loss": 0.15336619317531586, "cos_r_negg": [0.1263105571269989, 0.20785468816757202, 0.2468583583831787], "cos_innovation_negg": [0.5284179449081421, 0.5420424938201904, 0.43755000829696655], "cos_apical_negg": [0.1263105571269989, 0.20785468816757202, 0.2468583583831787], "cos_Ac_negg": [0.5282698273658752, 0.5419406890869141, 0.4372602105140686], "r_norm": [1.2138707637786865, 1.2036125659942627, 1.2089115381240845], "ldr": {"dL_sdil": -0.00979653000831604, "dL_bp": -0.008886262774467468, "ratio": 1.1024353270830574}}, {"step": 600, "epoch": 1, "train_loss": 0.12146282196044922, "cos_r_negg": [0.16558480262756348, 0.22119669616222382, 0.2640494406223297], "cos_innovation_negg": [0.5180521011352539, 0.5558491945266724, 0.4293135404586792], "cos_apical_negg": [0.16558480262756348, 0.22119669616222382, 0.2640494406223297], "cos_Ac_negg": [0.5179779529571533, 0.555755078792572, 0.42906081676483154], "r_norm": [1.1653333902359009, 1.1613284349441528, 1.1675937175750732]}, {"step": 700, "epoch": 1, "train_loss": 0.20129522681236267, "cos_r_negg": [0.16062965989112854, 0.24130436778068542, 0.3633827269077301], "cos_innovation_negg": [0.5213563442230225, 0.5475398302078247, 0.5486730337142944], "cos_apical_negg": [0.16062965989112854, 0.24130436778068542, 0.3633827269077301], "cos_Ac_negg": [0.5213738679885864, 0.5474972724914551, 0.5485571026802063], "r_norm": [1.1901230812072754, 1.1864830255508423, 1.195469856262207]}, {"step": 800, "epoch": 1, "train_loss": 0.15393279492855072, "cos_r_negg": [0.1316477656364441, 0.2005041241645813, 0.3116982877254486], "cos_innovation_negg": [0.49710673093795776, 0.5257840752601624, 0.5348391532897949], "cos_apical_negg": [0.1316477656364441, 0.2005041241645813, 0.3116982877254486], "cos_Ac_negg": [0.49711817502975464, 0.5257155299186707, 0.5347292423248291], "r_norm": [1.122615098953247, 1.1207232475280762, 1.1217429637908936]}, {"step": 900, "epoch": 1, "train_loss": 0.23652523756027222, "cos_r_negg": [0.1263885498046875, 0.2090131789445877, 0.36704394221305847], "cos_innovation_negg": [0.5136946439743042, 0.5036094188690186, 0.5819655656814575], "cos_apical_negg": [0.1263885498046875, 0.2090131789445877, 0.36704394221305847], "cos_Ac_negg": [0.513674259185791, 0.5035489797592163, 0.5818724632263184], "r_norm": [1.1491764783859253, 1.1485307216644287, 1.1522923707962036]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9476, "test_loss": 0.20409825744628907}, {"step": 1000, "epoch": 2, "train_loss": 0.04947963356971741, "cos_r_negg": [0.12906236946582794, 0.21162265539169312, 0.348416805267334], "cos_innovation_negg": [0.5171824097633362, 0.5449851751327515, 0.5746995806694031], "cos_apical_negg": [0.12906236946582794, 0.21162265539169312, 0.348416805267334], "cos_Ac_negg": [0.5171871781349182, 0.5448256134986877, 0.5745273232460022], "r_norm": [1.103270411491394, 1.0993584394454956, 1.1070058345794678], "ldr": {"dL_sdil": -0.007384568452835083, "dL_bp": -0.006825581192970276, "ratio": 1.0818959212499755}}, {"step": 1100, "epoch": 2, "train_loss": 0.26648539304733276, "cos_r_negg": [0.12517738342285156, 0.1890411078929901, 0.2794380187988281], "cos_innovation_negg": [0.5369072556495667, 0.558696985244751, 0.6034679412841797], "cos_apical_negg": [0.12517738342285156, 0.1890411078929901, 0.2794380187988281], "cos_Ac_negg": [0.5368977189064026, 0.5586187839508057, 0.6034163236618042], "r_norm": [1.1235891580581665, 1.1186692714691162, 1.1226366758346558]}, {"step": 1200, "epoch": 2, "train_loss": 0.2048262059688568, "cos_r_negg": [0.09771303832530975, 0.17599990963935852, 0.3345041275024414], "cos_innovation_negg": [0.48631739616394043, 0.5278089642524719, 0.5706388354301453], "cos_apical_negg": [0.09771303832530975, 0.17599990963935852, 0.3345041275024414], "cos_Ac_negg": [0.48631414771080017, 0.5277293920516968, 0.5705646276473999], "r_norm": [1.0964975357055664, 1.0948665142059326, 1.0999548435211182]}, {"step": 1300, "epoch": 2, "train_loss": 0.16357849538326263, "cos_r_negg": [0.11733831465244293, 0.19436469674110413, 0.314643919467926], "cos_innovation_negg": [0.5063409805297852, 0.5092393159866333, 0.5826326608657837], "cos_apical_negg": [0.11733831465244293, 0.19436469674110413, 0.314643919467926], "cos_Ac_negg": [0.5063400268554688, 0.5091994404792786, 0.5825886726379395], "r_norm": [1.120036244392395, 1.115511178970337, 1.1228621006011963]}, {"step": 1400, "epoch": 2, "train_loss": 0.15772321820259094, "cos_r_negg": [0.10442176461219788, 0.17081187665462494, 0.2702012062072754], "cos_innovation_negg": [0.46532905101776123, 0.5119856595993042, 0.4701986312866211], "cos_apical_negg": [0.10442176461219788, 0.17081187665462494, 0.2702012062072754], "cos_Ac_negg": [0.46533602476119995, 0.5119734406471252, 0.47009581327438354], "r_norm": [1.094272255897522, 1.09502375125885, 1.0983967781066895]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9551, "test_loss": 0.16394522399902345}, {"step": 1500, "epoch": 3, "train_loss": 0.10516754537820816, "cos_r_negg": [0.09480533003807068, 0.16188447177410126, 0.3394983112812042], "cos_innovation_negg": [0.4841820299625397, 0.49998190999031067, 0.6024752855300903], "cos_apical_negg": [0.09480533003807068, 0.16188447177410126, 0.3394983112812042], "cos_Ac_negg": [0.48415231704711914, 0.4999416470527649, 0.602357029914856], "r_norm": [1.082397222518921, 1.0830413103103638, 1.0838232040405273], "ldr": {"dL_sdil": -0.005336984992027283, "dL_bp": -0.005066409707069397, "ratio": 1.0534057252772786}}, {"step": 1600, "epoch": 3, "train_loss": 0.13073858618736267, "cos_r_negg": [0.09687574207782745, 0.17678995430469513, 0.32607996463775635], "cos_innovation_negg": [0.5109266042709351, 0.5232068300247192, 0.6100045442581177], "cos_apical_negg": [0.09687574207782745, 0.17678995430469513, 0.32607996463775635], "cos_Ac_negg": [0.510970950126648, 0.5232208967208862, 0.6101869344711304], "r_norm": [1.0868866443634033, 1.0870692729949951, 1.0859053134918213]}, {"step": 1700, "epoch": 3, "train_loss": 0.11934582144021988, "cos_r_negg": [0.0949309766292572, 0.18343903124332428, 0.38537925481796265], "cos_innovation_negg": [0.4793432950973511, 0.5229952931404114, 0.655630350112915], "cos_apical_negg": [0.0949309766292572, 0.18343903124332428, 0.38537925481796265], "cos_Ac_negg": [0.47924715280532837, 0.5230076313018799, 0.6553148031234741], "r_norm": [1.082822322845459, 1.083268642425537, 1.081310510635376]}, {"step": 1800, "epoch": 3, "train_loss": 0.05349261313676834, "cos_r_negg": [0.09097310900688171, 0.16451048851013184, 0.3121575713157654], "cos_innovation_negg": [0.4797059893608093, 0.5175971388816833, 0.5561882853507996], "cos_apical_negg": [0.09097310900688171, 0.16451048851013184, 0.3121575713157654], "cos_Ac_negg": [0.47961992025375366, 0.5175373554229736, 0.5560489892959595], "r_norm": [1.0815494060516357, 1.0842307806015015, 1.0889325141906738]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9597, "test_loss": 0.1567654396057129}, {"step": 1900, "epoch": 4, "train_loss": 0.049702320247888565, "cos_r_negg": [0.11094648391008377, 0.18373540043830872, 0.33276915550231934], "cos_innovation_negg": [0.5151636600494385, 0.5544989705085754, 0.672356367111206], "cos_apical_negg": [0.11094648391008377, 0.18373540043830872, 0.33276915550231934], "cos_Ac_negg": [0.515153169631958, 0.5545079708099365, 0.6723426580429077], "r_norm": [1.0771498680114746, 1.0746958255767822, 1.0793105363845825]}, {"step": 2000, "epoch": 4, "train_loss": 0.17489692568778992, "cos_r_negg": [0.08195360004901886, 0.1622200608253479, 0.3456461429595947], "cos_innovation_negg": [0.4352279007434845, 0.47315454483032227, 0.6348254680633545], "cos_apical_negg": [0.08195360004901886, 0.1622200608253479, 0.3456461429595947], "cos_Ac_negg": [0.4351407289505005, 0.47306156158447266, 0.6346690654754639], "r_norm": [1.072052001953125, 1.074843168258667, 1.0736905336380005], "ldr": {"dL_sdil": -0.006359875202178955, "dL_bp": -0.007607370615005493, "ratio": 0.8360149023940202}}, {"step": 2100, "epoch": 4, "train_loss": 0.04057953134179115, "cos_r_negg": [0.11039240658283234, 0.19588319957256317, 0.3653199374675751], "cos_innovation_negg": [0.49805307388305664, 0.5498154163360596, 0.6796514987945557], "cos_apical_negg": [0.11039240658283234, 0.19588319957256317, 0.3653199374675751], "cos_Ac_negg": [0.49799609184265137, 0.5497889518737793, 0.6795431971549988], "r_norm": [1.0667060613632202, 1.0698308944702148, 1.0696548223495483]}, {"step": 2200, "epoch": 4, "train_loss": 0.01804332807660103, "cos_r_negg": [0.08984159678220749, 0.14002031087875366, 0.29909420013427734], "cos_innovation_negg": [0.46091675758361816, 0.5379999279975891, 0.6175446510314941], "cos_apical_negg": [0.08984159678220749, 0.14002031087875366, 0.29909420013427734], "cos_Ac_negg": [0.460916668176651, 0.5380631685256958, 0.617724597454071], "r_norm": [1.038021445274353, 1.0354242324829102, 1.038496971130371]}, {"step": 2300, "epoch": 4, "train_loss": 0.24117830395698547, "cos_r_negg": [0.08873602747917175, 0.16129721701145172, 0.3125532865524292], "cos_innovation_negg": [0.4965319037437439, 0.5336123704910278, 0.6705626249313354], "cos_apical_negg": [0.08873602747917175, 0.16129721701145172, 0.3125532865524292], "cos_Ac_negg": [0.4963240623474121, 0.5335754156112671, 0.6705212593078613], "r_norm": [1.0376596450805664, 1.0370090007781982, 1.0377192497253418]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9611, "test_loss": 0.1636854202270508}, {"step": 2400, "epoch": 5, "train_loss": 0.10627827048301697, "cos_r_negg": [0.08160525560379028, 0.18309380114078522, 0.3644033372402191], "cos_innovation_negg": [0.4931505024433136, 0.5428751707077026, 0.6568996906280518], "cos_apical_negg": [0.08160525560379028, 0.18309380114078522, 0.3644033372402191], "cos_Ac_negg": [0.4930899441242218, 0.5427823066711426, 0.656874418258667], "r_norm": [1.0391044616699219, 1.0357418060302734, 1.0385117530822754]}, {"step": 2500, "epoch": 5, "train_loss": 0.04008999466896057, "cos_r_negg": [0.08401623368263245, 0.17791622877120972, 0.3076491951942444], "cos_innovation_negg": [0.4855118691921234, 0.5449755787849426, 0.665241539478302], "cos_apical_negg": [0.08401623368263245, 0.17791622877120972, 0.3076491951942444], "cos_Ac_negg": [0.48523539304733276, 0.5450470447540283, 0.664824366569519], "r_norm": [1.0601152181625366, 1.054722547531128, 1.0588864088058472], "ldr": {"dL_sdil": -0.007600918412208557, "dL_bp": -0.009574659168720245, "ratio": 0.793857857315719}}, {"step": 2600, "epoch": 5, "train_loss": 0.10991664230823517, "cos_r_negg": [0.08912836015224457, 0.16437111794948578, 0.28939080238342285], "cos_innovation_negg": [0.46123170852661133, 0.5317376852035522, 0.6545934677124023], "cos_apical_negg": [0.08912836015224457, 0.16437111794948578, 0.28939080238342285], "cos_Ac_negg": [0.4607233703136444, 0.5315687656402588, 0.6538490056991577], "r_norm": [1.0448853969573975, 1.0422736406326294, 1.0459226369857788]}, {"step": 2700, "epoch": 5, "train_loss": 0.06055522710084915, "cos_r_negg": [0.08246853202581406, 0.18223175406455994, 0.2698294520378113], "cos_innovation_negg": [0.4353107213973999, 0.509527325630188, 0.5890357494354248], "cos_apical_negg": [0.08246853202581406, 0.18223175406455994, 0.2698294520378113], "cos_Ac_negg": [0.4352387487888336, 0.5094600915908813, 0.5888159871101379], "r_norm": [1.0132813453674316, 1.0104095935821533, 1.010103464126587]}, {"step": 2800, "epoch": 5, "train_loss": 0.16425786912441254, "cos_r_negg": [0.08016721904277802, 0.17537474632263184, 0.38759416341781616], "cos_innovation_negg": [0.45793595910072327, 0.505504846572876, 0.6609646677970886], "cos_apical_negg": [0.08016721904277802, 0.17537474632263184, 0.38759416341781616], "cos_Ac_negg": [0.45793068408966064, 0.5054455995559692, 0.6608702540397644], "r_norm": [1.0355772972106934, 1.0334874391555786, 1.0356312990188599]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9619, "test_loss": 0.15686889038085938}, {"step": 2900, "epoch": 6, "train_loss": 0.022492146119475365, "cos_r_negg": [0.07196735590696335, 0.16925537586212158, 0.31764864921569824], "cos_innovation_negg": [0.45331889390945435, 0.5492408871650696, 0.6572692394256592], "cos_apical_negg": [0.07196735590696335, 0.16925537586212158, 0.31764864921569824], "cos_Ac_negg": [0.4532802700996399, 0.5492361783981323, 0.6572920680046082], "r_norm": [1.0519925355911255, 1.0493084192276, 1.052395224571228]}, {"step": 3000, "epoch": 6, "train_loss": 0.31273213028907776, "cos_r_negg": [0.0731772929430008, 0.13190704584121704, 0.3190533220767975], "cos_innovation_negg": [0.42692163586616516, 0.5165793895721436, 0.6689997315406799], "cos_apical_negg": [0.0731772929430008, 0.13190704584121704, 0.3190533220767975], "cos_Ac_negg": [0.4270516335964203, 0.5166670083999634, 0.6691550016403198], "r_norm": [1.0390095710754395, 1.036224126815796, 1.0383200645446777], "ldr": {"dL_sdil": -0.00616820901632309, "dL_bp": -0.0077553316950798035, "ratio": 0.7953507675547097}}, {"step": 3100, "epoch": 6, "train_loss": 0.1140340119600296, "cos_r_negg": [0.08143879473209381, 0.1412690281867981, 0.35382080078125], "cos_innovation_negg": [0.45451244711875916, 0.5342626571655273, 0.6729519367218018], "cos_apical_negg": [0.08143879473209381, 0.1412690281867981, 0.35382080078125], "cos_Ac_negg": [0.45452335476875305, 0.5344525575637817, 0.6728215217590332], "r_norm": [1.0448682308197021, 1.0408542156219482, 1.0420558452606201]}, {"step": 3200, "epoch": 6, "train_loss": 0.12365753948688507, "cos_r_negg": [0.07425111532211304, 0.13239648938179016, 0.24433907866477966], "cos_innovation_negg": [0.4392765164375305, 0.5060632228851318, 0.6366022825241089], "cos_apical_negg": [0.07425111532211304, 0.13239648938179016, 0.24433907866477966], "cos_Ac_negg": [0.43923044204711914, 0.5059811472892761, 0.6366878747940063], "r_norm": [0.9932649731636047, 0.9914728403091431, 0.9906747937202454]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9616, "test_loss": 0.1611527805328369}, {"step": 3300, "epoch": 7, "train_loss": 0.04431071877479553, "cos_r_negg": [0.07450416684150696, 0.1297498345375061, 0.264461487531662], "cos_innovation_negg": [0.42629730701446533, 0.5000718832015991, 0.5907208323478699], "cos_apical_negg": [0.07450416684150696, 0.1297498345375061, 0.264461487531662], "cos_Ac_negg": [0.4261389374732971, 0.5000936985015869, 0.5907834768295288], "r_norm": [1.0344806909561157, 1.032099962234497, 1.0320783853530884]}, {"step": 3400, "epoch": 7, "train_loss": 0.08375701308250427, "cos_r_negg": [0.09938184916973114, 0.12510553002357483, 0.2544529139995575], "cos_innovation_negg": [0.4636978209018707, 0.5547892451286316, 0.6818769574165344], "cos_apical_negg": [0.09938184916973114, 0.12510553002357483, 0.2544529139995575], "cos_Ac_negg": [0.4634917378425598, 0.5548191070556641, 0.6818888187408447], "r_norm": [1.0641742944717407, 1.059363842010498, 1.061361312866211]}, {"step": 3500, "epoch": 7, "train_loss": 0.09449511766433716, "cos_r_negg": [0.07759363949298859, 0.12673451006412506, 0.2399180829524994], "cos_innovation_negg": [0.4407230019569397, 0.5451507568359375, 0.6767064332962036], "cos_apical_negg": [0.07759363949298859, 0.12673451006412506, 0.2399180829524994], "cos_Ac_negg": [0.44026365876197815, 0.5451226830482483, 0.6768074035644531], "r_norm": [1.0044913291931152, 1.001854658126831, 1.0010911226272583], "ldr": {"dL_sdil": -0.004060566425323486, "dL_bp": -0.00534270703792572, "ratio": 0.7600204159612655}}, {"step": 3600, "epoch": 7, "train_loss": 0.014355139806866646, "cos_r_negg": [0.057094424962997437, 0.11132264882326126, 0.31315529346466064], "cos_innovation_negg": [0.4073660969734192, 0.509350061416626, 0.6994157433509827], "cos_apical_negg": [0.057094424962997437, 0.11132264882326126, 0.31315529346466064], "cos_Ac_negg": [0.4073219895362854, 0.5092644691467285, 0.6995916366577148], "r_norm": [1.0379538536071777, 1.0361063480377197, 1.0359954833984375]}, {"step": 3700, "epoch": 7, "train_loss": 0.026530861854553223, "cos_r_negg": [0.0712440088391304, 0.11641159653663635, 0.2877442538738251], "cos_innovation_negg": [0.4202393889427185, 0.5186985731124878, 0.6536667346954346], "cos_apical_negg": [0.0712440088391304, 0.11641159653663635, 0.2877442538738251], "cos_Ac_negg": [0.42011284828186035, 0.5187029838562012, 0.6535063982009888], "r_norm": [0.999384343624115, 0.9963240623474121, 0.9962331652641296]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9618, "test_loss": 0.16294672317504882}, {"step": 3800, "epoch": 8, "train_loss": 0.05007879436016083, "cos_r_negg": [0.07511164993047714, 0.14734423160552979, 0.3085169196128845], "cos_innovation_negg": [0.4413944482803345, 0.5397770404815674, 0.7106569409370422], "cos_apical_negg": [0.07511164993047714, 0.14734423160552979, 0.3085169196128845], "cos_Ac_negg": [0.441325306892395, 0.5396966934204102, 0.7107394337654114], "r_norm": [1.0129104852676392, 1.0108343362808228, 1.0100213289260864]}, {"step": 3900, "epoch": 8, "train_loss": 0.21742382645606995, "cos_r_negg": [0.09494712948799133, 0.13465934991836548, 0.2579788565635681], "cos_innovation_negg": [0.4363745152950287, 0.5148830413818359, 0.65738445520401], "cos_apical_negg": [0.09494712948799133, 0.13465934991836548, 0.2579788565635681], "cos_Ac_negg": [0.4360920190811157, 0.514843225479126, 0.6574424505233765], "r_norm": [1.0040419101715088, 1.0015568733215332, 1.0002267360687256]}, {"step": 4000, "epoch": 8, "train_loss": 0.07472498714923859, "cos_r_negg": [0.08299684524536133, 0.14769677817821503, 0.3467448949813843], "cos_innovation_negg": [0.42636168003082275, 0.5000577569007874, 0.6933121681213379], "cos_apical_negg": [0.08299684524536133, 0.14769677817821503, 0.3467448949813843], "cos_Ac_negg": [0.4264153838157654, 0.4998900592327118, 0.6923861503601074], "r_norm": [1.0153279304504395, 1.0113576650619507, 1.0112497806549072], "ldr": {"dL_sdil": -0.006744697690010071, "dL_bp": -0.008618451654911041, "ratio": 0.7825880981958921}}, {"step": 4100, "epoch": 8, "train_loss": 0.022548433393239975, "cos_r_negg": [0.07248678803443909, 0.10661035776138306, 0.3099792003631592], "cos_innovation_negg": [0.42672696709632874, 0.5190163254737854, 0.6978937387466431], "cos_apical_negg": [0.07248678803443909, 0.10661035776138306, 0.3099792003631592], "cos_Ac_negg": [0.42665842175483704, 0.5191322565078735, 0.6976729035377502], "r_norm": [0.994434654712677, 0.9916395545005798, 0.9928121566772461]}, {"step": 4200, "epoch": 8, "train_loss": 0.1103154793381691, "cos_r_negg": [0.07517226040363312, 0.10708636045455933, 0.3129129707813263], "cos_innovation_negg": [0.43011996150016785, 0.5349301695823669, 0.6778147220611572], "cos_apical_negg": [0.07517226040363312, 0.10708636045455933, 0.3129129707813263], "cos_Ac_negg": [0.43020758032798767, 0.5349271297454834, 0.6779068112373352], "r_norm": [0.9925261735916138, 0.9870480895042419, 0.9901551604270935]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9636, "test_loss": 0.1582131565093994}, {"step": 4300, "epoch": 9, "train_loss": 0.03350401669740677, "cos_r_negg": [0.07466697692871094, 0.10558614134788513, 0.3342083692550659], "cos_innovation_negg": [0.43360114097595215, 0.5500720739364624, 0.7179794311523438], "cos_apical_negg": [0.07466697692871094, 0.10558614134788513, 0.3342083692550659], "cos_Ac_negg": [0.43355754017829895, 0.5500990748405457, 0.7178323864936829], "r_norm": [0.9753940105438232, 0.9736157655715942, 0.9734360575675964]}, {"step": 4400, "epoch": 9, "train_loss": 0.029651187360286713, "cos_r_negg": [0.05542431026697159, 0.1084962785243988, 0.3036450147628784], "cos_innovation_negg": [0.44660431146621704, 0.535990834236145, 0.7104226350784302], "cos_apical_negg": [0.05542431026697159, 0.1084962785243988, 0.3036450147628784], "cos_Ac_negg": [0.4463743567466736, 0.5357836484909058, 0.7095645070075989], "r_norm": [1.0023603439331055, 0.9978123903274536, 0.9969149827957153]}, {"step": 4500, "epoch": 9, "train_loss": 0.13401556015014648, "cos_r_negg": [0.05591508373618126, 0.10642553865909576, 0.2879694104194641], "cos_innovation_negg": [0.4271180331707001, 0.5109951496124268, 0.6960757970809937], "cos_apical_negg": [0.05591508373618126, 0.10642553865909576, 0.2879694104194641], "cos_Ac_negg": [0.4278360903263092, 0.5115805268287659, 0.6965353488922119], "r_norm": [0.9934495091438293, 0.9916216731071472, 0.990249752998352], "ldr": {"dL_sdil": -0.004611022770404816, "dL_bp": -0.006804727017879486, "ratio": 0.6776205361786459}}, {"step": 4600, "epoch": 9, "train_loss": 0.020621519535779953, "cos_r_negg": [0.08068762719631195, 0.12873448431491852, 0.25776466727256775], "cos_innovation_negg": [0.43766605854034424, 0.5402704477310181, 0.7035529017448425], "cos_apical_negg": [0.08068762719631195, 0.12873448431491852, 0.25776466727256775], "cos_Ac_negg": [0.4376477003097534, 0.5403668880462646, 0.7036139965057373], "r_norm": [1.0251803398132324, 1.0206835269927979, 1.019230842590332]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9626, "test_loss": 0.16525814552307128}, {"step": 4700, "epoch": 10, "train_loss": 0.05971750617027283, "cos_r_negg": [0.08216692507266998, 0.11701570451259613, 0.26269209384918213], "cos_innovation_negg": [0.41004735231399536, 0.5183526873588562, 0.6850098371505737], "cos_apical_negg": [0.08216692507266998, 0.11701570451259613, 0.26269209384918213], "cos_Ac_negg": [0.4099718928337097, 0.5187126398086548, 0.6854265928268433], "r_norm": [1.0344624519348145, 1.0298960208892822, 1.028477668762207]}, {"step": 4800, "epoch": 10, "train_loss": 0.03930007666349411, "cos_r_negg": [0.0657869502902031, 0.10045967251062393, 0.20310693979263306], "cos_innovation_negg": [0.3980203866958618, 0.5154334306716919, 0.6973041296005249], "cos_apical_negg": [0.0657869502902031, 0.10045967251062393, 0.20310693979263306], "cos_Ac_negg": [0.3977644443511963, 0.5152593851089478, 0.6979318857192993], "r_norm": [1.0008891820907593, 0.9972832202911377, 0.9957632422447205]}, {"step": 4900, "epoch": 10, "train_loss": 0.05141081660985947, "cos_r_negg": [0.09262190759181976, 0.0856299102306366, 0.22405169904232025], "cos_innovation_negg": [0.39689740538597107, 0.4948880076408386, 0.6602343320846558], "cos_apical_negg": [0.09262190759181976, 0.0856299102306366, 0.22405169904232025], "cos_Ac_negg": [0.39682963490486145, 0.49598509073257446, 0.6607551574707031], "r_norm": [1.0169191360473633, 1.0117501020431519, 1.0106686353683472]}, {"step": 5000, "epoch": 10, "train_loss": 0.0069166868925094604, "cos_r_negg": [0.09611442685127258, 0.09031155705451965, 0.22948533296585083], "cos_innovation_negg": [0.4434712827205658, 0.5348515510559082, 0.6943167448043823], "cos_apical_negg": [0.09611442685127258, 0.09031155705451965, 0.22948533296585083], "cos_Ac_negg": [0.44347018003463745, 0.5350801944732666, 0.6944035291671753], "r_norm": [1.0307512283325195, 1.0238075256347656, 1.025500774383545], "ldr": {"dL_sdil": -0.008185997605323792, "dL_bp": -0.012424886226654053, "ratio": 0.6588388381185388}}, {"step": 5100, "epoch": 10, "train_loss": 0.11458219587802887, "cos_r_negg": [0.07597172260284424, 0.12591536343097687, 0.20473456382751465], "cos_innovation_negg": [0.4306093454360962, 0.526761531829834, 0.7296078205108643], "cos_apical_negg": [0.07597172260284424, 0.12591536343097687, 0.20473456382751465], "cos_Ac_negg": [0.4304288625717163, 0.5267440676689148, 0.729908287525177], "r_norm": [1.0243782997131348, 1.0203747749328613, 1.0188733339309692]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9617, "test_loss": 0.16517740592956542}, {"step": 5200, "epoch": 11, "train_loss": 0.010156534612178802, "cos_r_negg": [0.08489933609962463, 0.09461561590433121, 0.2719295620918274], "cos_innovation_negg": [0.41963574290275574, 0.5246255993843079, 0.7058186531066895], "cos_apical_negg": [0.08489933609962463, 0.09461561590433121, 0.2719295620918274], "cos_Ac_negg": [0.4194912612438202, 0.5246355533599854, 0.7058523893356323], "r_norm": [1.0145225524902344, 1.0102989673614502, 1.0099711418151855]}, {"step": 5300, "epoch": 11, "train_loss": 0.05159877613186836, "cos_r_negg": [0.07125509530305862, 0.0836147889494896, 0.24942587316036224], "cos_innovation_negg": [0.39580056071281433, 0.48831963539123535, 0.7069228887557983], "cos_apical_negg": [0.07125509530305862, 0.0836147889494896, 0.24942587316036224], "cos_Ac_negg": [0.39568838477134705, 0.4883936941623688, 0.7068790197372437], "r_norm": [0.9980209469795227, 0.9941141605377197, 0.9908428192138672]}, {"step": 5400, "epoch": 11, "train_loss": 0.10683313757181168, "cos_r_negg": [0.06005481258034706, 0.11635561287403107, 0.2433566004037857], "cos_innovation_negg": [0.4024996757507324, 0.4987344741821289, 0.7048144340515137], "cos_apical_negg": [0.06005481258034706, 0.11635561287403107, 0.2433566004037857], "cos_Ac_negg": [0.40215033292770386, 0.49826884269714355, 0.7049583196640015], "r_norm": [0.9942977428436279, 0.9889447689056396, 0.9858816862106323]}, {"step": 5500, "epoch": 11, "train_loss": 0.060216888785362244, "cos_r_negg": [0.036806873977184296, 0.06755854189395905, 0.21856330335140228], "cos_innovation_negg": [0.39783596992492676, 0.5225698947906494, 0.6927446722984314], "cos_apical_negg": [0.036806873977184296, 0.06755854189395905, 0.21856330335140228], "cos_Ac_negg": [0.39800986647605896, 0.5231701135635376, 0.6930423974990845], "r_norm": [1.0041204690933228, 0.9986884593963623, 0.9972342252731323], "ldr": {"dL_sdil": -0.005161933600902557, "dL_bp": -0.0078410804271698, "ratio": 0.6583191753799843}}, {"step": 5600, "epoch": 11, "train_loss": 0.022681772708892822, "cos_r_negg": [0.06077684834599495, 0.08140318840742111, 0.23175372183322906], "cos_innovation_negg": [0.4105914235115051, 0.5120735168457031, 0.7293616533279419], "cos_apical_negg": [0.06077684834599495, 0.08140318840742111, 0.23175372183322906], "cos_Ac_negg": [0.4105708599090576, 0.512154221534729, 0.7305968999862671], "r_norm": [1.0110012292861938, 1.0048599243164062, 1.0039482116699219]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9622, "test_loss": 0.17614651985168456}, {"step": 5700, "epoch": 12, "train_loss": 0.03661233186721802, "cos_r_negg": [0.056666113436222076, 0.103221595287323, 0.2308501899242401], "cos_innovation_negg": [0.39536982774734497, 0.5059580206871033, 0.7346677184104919], "cos_apical_negg": [0.056666113436222076, 0.103221595287323, 0.2308501899242401], "cos_Ac_negg": [0.3956769108772278, 0.5059629082679749, 0.7349888682365417], "r_norm": [1.0067126750946045, 1.002601146697998, 1.0002315044403076]}, {"step": 5800, "epoch": 12, "train_loss": 0.015466328710317612, "cos_r_negg": [0.06751728802919388, 0.10087569057941437, 0.2331043928861618], "cos_innovation_negg": [0.3826684355735779, 0.5069130659103394, 0.726081132888794], "cos_apical_negg": [0.06751728802919388, 0.10087569057941437, 0.2331043928861618], "cos_Ac_negg": [0.3827116787433624, 0.5069876313209534, 0.7267923355102539], "r_norm": [1.0080156326293945, 1.002086877822876, 1.0011829137802124]}, {"step": 5900, "epoch": 12, "train_loss": 0.03919490426778793, "cos_r_negg": [0.062435634434223175, 0.08189548552036285, 0.2416512668132782], "cos_innovation_negg": [0.40043920278549194, 0.4908711016178131, 0.7183642387390137], "cos_apical_negg": [0.062435634434223175, 0.08189548552036285, 0.2416512668132782], "cos_Ac_negg": [0.4007493555545807, 0.49157220125198364, 0.7197014093399048], "r_norm": [1.017671823501587, 1.010943055152893, 1.0096156597137451]}, {"step": 6000, "epoch": 12, "train_loss": 0.010215059854090214, "cos_r_negg": [0.06374108046293259, 0.08141080290079117, 0.2767689824104309], "cos_innovation_negg": [0.4110422134399414, 0.4929632246494293, 0.6879470348358154], "cos_apical_negg": [0.06374108046293259, 0.08141080290079117, 0.2767689824104309], "cos_Ac_negg": [0.41087836027145386, 0.49321430921554565, 0.6877579689025879], "r_norm": [0.9880693554878235, 0.9828629493713379, 0.9834429025650024], "ldr": {"dL_sdil": -0.003598034381866455, "dL_bp": -0.006107695400714874, "ratio": 0.5890985299373842}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9635, "test_loss": 0.1686151653289795}, {"step": 6100, "epoch": 13, "train_loss": 0.035873670130968094, "cos_r_negg": [0.0469224750995636, 0.08607562631368637, 0.25058436393737793], "cos_innovation_negg": [0.38837969303131104, 0.4926930367946625, 0.6913175582885742], "cos_apical_negg": [0.0469224750995636, 0.08607562631368637, 0.25058436393737793], "cos_Ac_negg": [0.38867029547691345, 0.492725133895874, 0.6915112137794495], "r_norm": [1.0014129877090454, 0.9950658082962036, 0.9941937923431396]}, {"step": 6200, "epoch": 13, "train_loss": 0.016559874638915062, "cos_r_negg": [0.0659569799900055, 0.07672332227230072, 0.20249177515506744], "cos_innovation_negg": [0.3988562822341919, 0.4817458391189575, 0.6683967113494873], "cos_apical_negg": [0.0659569799900055, 0.07672332227230072, 0.20249177515506744], "cos_Ac_negg": [0.3983447253704071, 0.48185744881629944, 0.6681798696517944], "r_norm": [1.0086009502410889, 1.0023517608642578, 0.9995269775390625]}, {"step": 6300, "epoch": 13, "train_loss": 0.01982077583670616, "cos_r_negg": [0.06806819885969162, 0.09169220924377441, 0.20339035987854004], "cos_innovation_negg": [0.39540067315101624, 0.5070501565933228, 0.7026590704917908], "cos_apical_negg": [0.06806819885969162, 0.09169220924377441, 0.20339035987854004], "cos_Ac_negg": [0.3948124051094055, 0.5073511600494385, 0.7030317187309265], "r_norm": [1.0116000175476074, 1.004624843597412, 1.002839207649231]}, {"step": 6400, "epoch": 13, "train_loss": 0.06360478699207306, "cos_r_negg": [0.0648370310664177, 0.10829947888851166, 0.19740301370620728], "cos_innovation_negg": [0.37756043672561646, 0.5060795545578003, 0.7141621112823486], "cos_apical_negg": [0.0648370310664177, 0.10829947888851166, 0.19740301370620728], "cos_Ac_negg": [0.3773195147514343, 0.5059973001480103, 0.7144060730934143], "r_norm": [0.9938490390777588, 0.9881103038787842, 0.9874942302703857]}, {"step": 6500, "epoch": 13, "train_loss": 0.005917523987591267, "cos_r_negg": [0.05767172947525978, 0.10952644050121307, 0.2132006585597992], "cos_innovation_negg": [0.40701770782470703, 0.5423383712768555, 0.6835932731628418], "cos_apical_negg": [0.05767172947525978, 0.10952644050121307, 0.2132006585597992], "cos_Ac_negg": [0.40654534101486206, 0.5420957803726196, 0.6837042570114136], "r_norm": [1.0033890008926392, 0.9967707395553589, 0.9964233636856079], "ldr": {"dL_sdil": -0.005892999470233917, "dL_bp": -0.011221535503864288, "ratio": 0.5251509000889034}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9635, "test_loss": 0.1701450241088867}, {"step": 6600, "epoch": 14, "train_loss": 0.01692129671573639, "cos_r_negg": [0.047284916043281555, 0.10773734748363495, 0.20169442892074585], "cos_innovation_negg": [0.3913191556930542, 0.5143514275550842, 0.7084053158760071], "cos_apical_negg": [0.047284916043281555, 0.10773734748363495, 0.20169442892074585], "cos_Ac_negg": [0.3909500241279602, 0.5146305561065674, 0.7091703414916992], "r_norm": [0.9986156225204468, 0.9927716255187988, 0.9917881488800049]}, {"step": 6700, "epoch": 14, "train_loss": 0.07555331289768219, "cos_r_negg": [0.05310492590069771, 0.07415593415498734, 0.1699831634759903], "cos_innovation_negg": [0.4001525640487671, 0.5247504115104675, 0.6986910104751587], "cos_apical_negg": [0.05310492590069771, 0.07415593415498734, 0.1699831634759903], "cos_Ac_negg": [0.3991778492927551, 0.5261366367340088, 0.7006081342697144], "r_norm": [0.9993978142738342, 0.993030846118927, 0.993079423904419]}, {"step": 6800, "epoch": 14, "train_loss": 0.03224220499396324, "cos_r_negg": [0.05489124730229378, 0.10033617913722992, 0.1890159547328949], "cos_innovation_negg": [0.416041761636734, 0.5418425798416138, 0.6230579614639282], "cos_apical_negg": [0.05489124730229378, 0.10033617913722992, 0.1890159547328949], "cos_Ac_negg": [0.4159555435180664, 0.5421961545944214, 0.6224014759063721], "r_norm": [0.995155930519104, 0.9891947507858276, 0.9877503514289856]}, {"step": 6900, "epoch": 14, "train_loss": 0.022383030503988266, "cos_r_negg": [0.04361272230744362, 0.09534022212028503, 0.20757703483104706], "cos_innovation_negg": [0.4027337431907654, 0.5326934456825256, 0.6694039106369019], "cos_apical_negg": [0.04361272230744362, 0.09534022212028503, 0.20757703483104706], "cos_Ac_negg": [0.4019620418548584, 0.5330390930175781, 0.6700315475463867], "r_norm": [1.0033528804779053, 0.9959549307823181, 0.9956663250923157]}, {"step": 7000, "epoch": 14, "train_loss": 0.039209477603435516, "cos_r_negg": [0.05443475395441055, 0.09657007455825806, 0.18467269837856293], "cos_innovation_negg": [0.37617555260658264, 0.5200367569923401, 0.6829506754875183], "cos_apical_negg": [0.05443475395441055, 0.09657007455825806, 0.18467269837856293], "cos_Ac_negg": [0.37569886445999146, 0.5202019214630127, 0.6829924583435059], "r_norm": [1.0039644241333008, 0.9969807863235474, 0.9963716864585876], "ldr": {"dL_sdil": -0.004821017384529114, "dL_bp": -0.007645532488822937, "ratio": 0.630566594488611}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.964, "test_loss": 0.1758670551300049}], "final": {"test_acc": 0.964, "test_loss": 0.1758670551300049, "wall_s": 33.141804695129395, "cos_r_negg": [0.055599045008420944, 0.09485368430614471, 0.19291341304779053], "cos_innovation_negg": [0.3823274075984955, 0.5083365440368652, 0.6721898317337036], "cos_apical_negg": [0.055599045008420944, 0.09485368430614471, 0.19291341304779053], "cos_Ac_negg": [0.3818332850933075, 0.5084835886955261, 0.6726865768432617]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_raw_s4.json b/results/nuis_ctrl_v1_rho0p05_raw_s4.json new file mode 100644 index 0000000..a138ad4 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_raw_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_raw_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.01276855356991291, 0.027974901720881462, 0.007863546721637249], "cos_innovation_negg": [-0.012265687808394432, 0.028612103313207626, 0.008681844919919968], "cos_apical_negg": [-0.01276855356991291, 0.027974901720881462, 0.007863546721637249], "cos_Ac_negg": [-0.012265857309103012, 0.028612008318305016, 0.008681772276759148], "r_norm": [4.9412102699279785, 4.886099338531494, 4.965764999389648], "ldr": {"dL_sdil": -0.002613544464111328, "dL_bp": -0.2646901607513428, "ratio": 0.00987397664005563}}, {"step": 100, "epoch": 0, "train_loss": 0.26013994216918945, "cos_r_negg": [0.24850524961948395, 0.25191164016723633, 0.272588849067688], "cos_innovation_negg": [0.5308746099472046, 0.47673410177230835, 0.37461376190185547], "cos_apical_negg": [0.24850524961948395, 0.25191164016723633, 0.272588849067688], "cos_Ac_negg": [0.5308569073677063, 0.4768475890159607, 0.37450483441352844], "r_norm": [1.5276544094085693, 1.5313708782196045, 1.545727014541626]}, {"step": 200, "epoch": 0, "train_loss": 0.29375600814819336, "cos_r_negg": [0.2392352819442749, 0.2733278274536133, 0.27691513299942017], "cos_innovation_negg": [0.5437605381011963, 0.5081994533538818, 0.4157235026359558], "cos_apical_negg": [0.2392352819442749, 0.2733278274536133, 0.27691513299942017], "cos_Ac_negg": [0.5438199639320374, 0.5080227851867676, 0.4157814383506775], "r_norm": [1.418898344039917, 1.4281047582626343, 1.4369816780090332]}, {"step": 300, "epoch": 0, "train_loss": 0.2932523488998413, "cos_r_negg": [0.23650336265563965, 0.27414965629577637, 0.3194761872291565], "cos_innovation_negg": [0.5758167505264282, 0.5769020318984985, 0.4987916648387909], "cos_apical_negg": [0.23650336265563965, 0.27414965629577637, 0.3194761872291565], "cos_Ac_negg": [0.5758305788040161, 0.5769277811050415, 0.49880701303482056], "r_norm": [1.3357043266296387, 1.345266342163086, 1.3534455299377441]}, {"step": 400, "epoch": 0, "train_loss": 0.1698438823223114, "cos_r_negg": [0.16553497314453125, 0.22581245005130768, 0.3293743133544922], "cos_innovation_negg": [0.5541159510612488, 0.5609233379364014, 0.5268353819847107], "cos_apical_negg": [0.16553497314453125, 0.22581245005130768, 0.3293743133544922], "cos_Ac_negg": [0.5540916919708252, 0.5609796643257141, 0.526884138584137], "r_norm": [1.2621896266937256, 1.2686779499053955, 1.2758424282073975]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9405, "test_loss": 0.21326016387939453}, {"step": 500, "epoch": 1, "train_loss": 0.15398839116096497, "cos_r_negg": [0.1688903570175171, 0.22117003798484802, 0.2884068489074707], "cos_innovation_negg": [0.5467410087585449, 0.5154349207878113, 0.4775344133377075], "cos_apical_negg": [0.1688903570175171, 0.22117003798484802, 0.2884068489074707], "cos_Ac_negg": [0.5467312335968018, 0.5154768228530884, 0.4773128926753998], "r_norm": [1.2117657661437988, 1.2178266048431396, 1.229022741317749], "ldr": {"dL_sdil": -0.010761469602584839, "dL_bp": -0.008880361914634705, "ratio": 1.2118278180588673}}, {"step": 600, "epoch": 1, "train_loss": 0.09925056993961334, "cos_r_negg": [0.16322532296180725, 0.19497591257095337, 0.32837289571762085], "cos_innovation_negg": [0.5066546201705933, 0.489335298538208, 0.49765563011169434], "cos_apical_negg": [0.16322532296180725, 0.19497591257095337, 0.32837289571762085], "cos_Ac_negg": [0.5065829753875732, 0.4891866445541382, 0.497527152299881], "r_norm": [1.1727622747421265, 1.1827952861785889, 1.1884406805038452]}, {"step": 700, "epoch": 1, "train_loss": 0.15494240820407867, "cos_r_negg": [0.1633482575416565, 0.20705632865428925, 0.35798266530036926], "cos_innovation_negg": [0.521158754825592, 0.5058408975601196, 0.5488661527633667], "cos_apical_negg": [0.1633482575416565, 0.20705632865428925, 0.35798266530036926], "cos_Ac_negg": [0.5211193561553955, 0.5058815479278564, 0.5487911701202393], "r_norm": [1.1890515089035034, 1.1969449520111084, 1.202889084815979]}, {"step": 800, "epoch": 1, "train_loss": 0.13286997377872467, "cos_r_negg": [0.13530421257019043, 0.20170271396636963, 0.340335488319397], "cos_innovation_negg": [0.546042263507843, 0.5575150847434998, 0.5765976905822754], "cos_apical_negg": [0.13530421257019043, 0.20170271396636963, 0.340335488319397], "cos_Ac_negg": [0.546067476272583, 0.5574982166290283, 0.5764522552490234], "r_norm": [1.1382240056991577, 1.142512321472168, 1.1514114141464233]}, {"step": 900, "epoch": 1, "train_loss": 0.24774156510829926, "cos_r_negg": [0.1709466427564621, 0.2295072227716446, 0.38772282004356384], "cos_innovation_negg": [0.5662651062011719, 0.5407345294952393, 0.5741195678710938], "cos_apical_negg": [0.1709466427564621, 0.2295072227716446, 0.38772282004356384], "cos_Ac_negg": [0.5662455558776855, 0.5406302213668823, 0.573854923248291], "r_norm": [1.129356861114502, 1.131345272064209, 1.1407878398895264]}, {"epoch_end": 1, "step": 938, "test_acc": 0.944, "test_loss": 0.210464591217041}, {"step": 1000, "epoch": 2, "train_loss": 0.05530806630849838, "cos_r_negg": [0.1402783989906311, 0.20164531469345093, 0.38454777002334595], "cos_innovation_negg": [0.5203700065612793, 0.5064951777458191, 0.5947195887565613], "cos_apical_negg": [0.1402783989906311, 0.20164531469345093, 0.38454777002334595], "cos_Ac_negg": [0.5202797055244446, 0.5063457489013672, 0.5945744514465332], "r_norm": [1.1265949010849, 1.1327511072158813, 1.1379094123840332], "ldr": {"dL_sdil": -0.00884103775024414, "dL_bp": -0.008573353290557861, "ratio": 1.0312228425231338}}, {"step": 1100, "epoch": 2, "train_loss": 0.18691973388195038, "cos_r_negg": [0.13471540808677673, 0.17767074704170227, 0.39217913150787354], "cos_innovation_negg": [0.5172616839408875, 0.47908714413642883, 0.6195251941680908], "cos_apical_negg": [0.13471540808677673, 0.17767074704170227, 0.39217913150787354], "cos_Ac_negg": [0.5172070264816284, 0.4790244996547699, 0.6194561123847961], "r_norm": [1.1021387577056885, 1.1059601306915283, 1.1143887042999268]}, {"step": 1200, "epoch": 2, "train_loss": 0.2067277580499649, "cos_r_negg": [0.113346166908741, 0.1947312355041504, 0.389112651348114], "cos_innovation_negg": [0.521289050579071, 0.5408700108528137, 0.6286255717277527], "cos_apical_negg": [0.113346166908741, 0.1947312355041504, 0.389112651348114], "cos_Ac_negg": [0.5212043523788452, 0.5406849384307861, 0.6283817887306213], "r_norm": [1.0992424488067627, 1.1022663116455078, 1.1104168891906738]}, {"step": 1300, "epoch": 2, "train_loss": 0.1622488796710968, "cos_r_negg": [0.1405969262123108, 0.19370582699775696, 0.392331600189209], "cos_innovation_negg": [0.46598413586616516, 0.4714345932006836, 0.5880535840988159], "cos_apical_negg": [0.1405969262123108, 0.19370582699775696, 0.392331600189209], "cos_Ac_negg": [0.46584635972976685, 0.471155047416687, 0.5878466963768005], "r_norm": [1.1388928890228271, 1.1448814868927002, 1.1534709930419922]}, {"step": 1400, "epoch": 2, "train_loss": 0.16302144527435303, "cos_r_negg": [0.12247227132320404, 0.2063058316707611, 0.37510859966278076], "cos_innovation_negg": [0.49395352602005005, 0.5141550898551941, 0.6139331459999084], "cos_apical_negg": [0.12247227132320404, 0.2063058316707611, 0.37510859966278076], "cos_Ac_negg": [0.4937470555305481, 0.5138537883758545, 0.6137375831604004], "r_norm": [1.0973784923553467, 1.1018210649490356, 1.105970859527588]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9578, "test_loss": 0.15663828811645508}, {"step": 1500, "epoch": 3, "train_loss": 0.0855225920677185, "cos_r_negg": [0.12679940462112427, 0.187379851937294, 0.41415488719940186], "cos_innovation_negg": [0.46263599395751953, 0.4583713412284851, 0.612302303314209], "cos_apical_negg": [0.12679940462112427, 0.187379851937294, 0.41415488719940186], "cos_Ac_negg": [0.4625742435455322, 0.4581865668296814, 0.6120743155479431], "r_norm": [1.0643413066864014, 1.069561243057251, 1.0730834007263184], "ldr": {"dL_sdil": -0.004560746252536774, "dL_bp": -0.004647597670555115, "ratio": 0.9813126212347104}}, {"step": 1600, "epoch": 3, "train_loss": 0.11481274664402008, "cos_r_negg": [0.10856539756059647, 0.1522957980632782, 0.43272650241851807], "cos_innovation_negg": [0.4990244507789612, 0.5282912254333496, 0.6558042168617249], "cos_apical_negg": [0.10856539756059647, 0.1522957980632782, 0.43272650241851807], "cos_Ac_negg": [0.49895843863487244, 0.5281450748443604, 0.6539133787155151], "r_norm": [1.0802571773529053, 1.0838298797607422, 1.089996576309204]}, {"step": 1700, "epoch": 3, "train_loss": 0.09685763716697693, "cos_r_negg": [0.14557063579559326, 0.18873047828674316, 0.4243680536746979], "cos_innovation_negg": [0.49350735545158386, 0.5314773321151733, 0.6354253888130188], "cos_apical_negg": [0.14557063579559326, 0.18873047828674316, 0.4243680536746979], "cos_Ac_negg": [0.4933376908302307, 0.5312278270721436, 0.6353346705436707], "r_norm": [1.0909192562103271, 1.0945215225219727, 1.1035640239715576]}, {"step": 1800, "epoch": 3, "train_loss": 0.06086686998605728, "cos_r_negg": [0.10004259645938873, 0.20091745257377625, 0.4032532572746277], "cos_innovation_negg": [0.4952639639377594, 0.5358074903488159, 0.6659960746765137], "cos_apical_negg": [0.10004259645938873, 0.20091745257377625, 0.4032532572746277], "cos_Ac_negg": [0.4951687753200531, 0.5353055596351624, 0.6658689975738525], "r_norm": [1.0584827661514282, 1.0594239234924316, 1.0661325454711914]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9557, "test_loss": 0.17649168395996093}, {"step": 1900, "epoch": 4, "train_loss": 0.07546502351760864, "cos_r_negg": [0.10957294702529907, 0.20830683410167694, 0.39079493284225464], "cos_innovation_negg": [0.5019330382347107, 0.5308883786201477, 0.6334292888641357], "cos_apical_negg": [0.10957294702529907, 0.20830683410167694, 0.39079493284225464], "cos_Ac_negg": [0.5017940402030945, 0.5305935144424438, 0.6332826614379883], "r_norm": [1.082535982131958, 1.0822274684906006, 1.0914366245269775]}, {"step": 2000, "epoch": 4, "train_loss": 0.12586073577404022, "cos_r_negg": [0.08955678343772888, 0.18430209159851074, 0.3933820128440857], "cos_innovation_negg": [0.49093759059906006, 0.5192382335662842, 0.6256149411201477], "cos_apical_negg": [0.08955678343772888, 0.18430209159851074, 0.3933820128440857], "cos_Ac_negg": [0.4909134805202484, 0.5187180042266846, 0.6252263188362122], "r_norm": [1.0552659034729004, 1.0591601133346558, 1.0642681121826172], "ldr": {"dL_sdil": -0.006847351789474487, "dL_bp": -0.007140249013900757, "ratio": 0.9589794104020669}}, {"step": 2100, "epoch": 4, "train_loss": 0.035429831594228745, "cos_r_negg": [0.119143545627594, 0.1786750853061676, 0.4113121032714844], "cos_innovation_negg": [0.5101559162139893, 0.5627262592315674, 0.6266975402832031], "cos_apical_negg": [0.119143545627594, 0.1786750853061676, 0.4113121032714844], "cos_Ac_negg": [0.5101614594459534, 0.5625768899917603, 0.6265712976455688], "r_norm": [1.059920072555542, 1.0629360675811768, 1.0702342987060547]}, {"step": 2200, "epoch": 4, "train_loss": 0.033365048468112946, "cos_r_negg": [0.08176523447036743, 0.16746781766414642, 0.36795932054519653], "cos_innovation_negg": [0.4932630956172943, 0.5718141794204712, 0.6306283473968506], "cos_apical_negg": [0.08176523447036743, 0.16746781766414642, 0.36795932054519653], "cos_Ac_negg": [0.4932239055633545, 0.571699321269989, 0.6305181384086609], "r_norm": [1.0574030876159668, 1.0599828958511353, 1.064523696899414]}, {"step": 2300, "epoch": 4, "train_loss": 0.25615012645721436, "cos_r_negg": [0.08106667548418045, 0.1829918473958969, 0.4243144690990448], "cos_innovation_negg": [0.48078063130378723, 0.5395497679710388, 0.6645500659942627], "cos_apical_negg": [0.08106667548418045, 0.1829918473958969, 0.4243144690990448], "cos_Ac_negg": [0.48067957162857056, 0.5393576622009277, 0.6646496057510376], "r_norm": [1.0397179126739502, 1.042352557182312, 1.0465013980865479]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9616, "test_loss": 0.15654121017456055}, {"step": 2400, "epoch": 5, "train_loss": 0.14259488880634308, "cos_r_negg": [0.0766877606511116, 0.1229669600725174, 0.35789403319358826], "cos_innovation_negg": [0.4500814974308014, 0.4828101098537445, 0.6296756267547607], "cos_apical_negg": [0.0766877606511116, 0.1229669600725174, 0.35789403319358826], "cos_Ac_negg": [0.45002585649490356, 0.4827505946159363, 0.629584550857544], "r_norm": [1.0328072309494019, 1.035341739654541, 1.038388729095459]}, {"step": 2500, "epoch": 5, "train_loss": 0.045146387070417404, "cos_r_negg": [0.0641186460852623, 0.14930087327957153, 0.3463379144668579], "cos_innovation_negg": [0.45168396830558777, 0.5197337865829468, 0.6641145944595337], "cos_apical_negg": [0.0641186460852623, 0.14930087327957153, 0.3463379144668579], "cos_Ac_negg": [0.45177531242370605, 0.5193834900856018, 0.6637856364250183], "r_norm": [1.0509676933288574, 1.0534470081329346, 1.05649733543396], "ldr": {"dL_sdil": -0.0070339590311050415, "dL_bp": -0.007535845041275024, "ratio": 0.9334001684720064}}, {"step": 2600, "epoch": 5, "train_loss": 0.10692869126796722, "cos_r_negg": [0.08029148727655411, 0.15471167862415314, 0.3597244620323181], "cos_innovation_negg": [0.4552716016769409, 0.5071496963500977, 0.6587305068969727], "cos_apical_negg": [0.08029148727655411, 0.15471167862415314, 0.3597244620323181], "cos_Ac_negg": [0.45515599846839905, 0.506893515586853, 0.6586518287658691], "r_norm": [1.0370371341705322, 1.0405818223953247, 1.042618989944458]}, {"step": 2700, "epoch": 5, "train_loss": 0.057580314576625824, "cos_r_negg": [0.11116532981395721, 0.17247194051742554, 0.3980430066585541], "cos_innovation_negg": [0.4248823821544647, 0.4200972020626068, 0.6716198921203613], "cos_apical_negg": [0.11116532981395721, 0.17247194051742554, 0.3980430066585541], "cos_Ac_negg": [0.4248430132865906, 0.4199381470680237, 0.6715167164802551], "r_norm": [1.0334545373916626, 1.0371612310409546, 1.0430705547332764]}, {"step": 2800, "epoch": 5, "train_loss": 0.14944960176944733, "cos_r_negg": [0.09190257638692856, 0.17375566065311432, 0.3027140200138092], "cos_innovation_negg": [0.47083961963653564, 0.4981519281864166, 0.6407217979431152], "cos_apical_negg": [0.09190257638692856, 0.17375566065311432, 0.3027140200138092], "cos_Ac_negg": [0.4708113670349121, 0.49773240089416504, 0.6406062245368958], "r_norm": [1.0166034698486328, 1.0176409482955933, 1.0220248699188232]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9605, "test_loss": 0.15841765823364257}, {"step": 2900, "epoch": 6, "train_loss": 0.07720599323511124, "cos_r_negg": [0.10671725869178772, 0.17564380168914795, 0.3285195827484131], "cos_innovation_negg": [0.470671683549881, 0.5171830654144287, 0.655055582523346], "cos_apical_negg": [0.10671725869178772, 0.17564380168914795, 0.3285195827484131], "cos_Ac_negg": [0.47066807746887207, 0.5167785882949829, 0.6550564765930176], "r_norm": [1.0551540851593018, 1.0575717687606812, 1.0610696077346802]}, {"step": 3000, "epoch": 6, "train_loss": 0.2923441231250763, "cos_r_negg": [0.090237095952034, 0.11580106616020203, 0.3385462164878845], "cos_innovation_negg": [0.436171293258667, 0.4663727581501007, 0.6474610567092896], "cos_apical_negg": [0.090237095952034, 0.11580106616020203, 0.3385462164878845], "cos_Ac_negg": [0.43597984313964844, 0.46631571650505066, 0.6472241878509521], "r_norm": [1.0401558876037598, 1.0426424741744995, 1.04593825340271], "ldr": {"dL_sdil": -0.007276475429534912, "dL_bp": -0.008799374103546143, "ratio": 0.8269310230374791}}, {"step": 3100, "epoch": 6, "train_loss": 0.13063712418079376, "cos_r_negg": [0.09154856950044632, 0.15246111154556274, 0.36129021644592285], "cos_innovation_negg": [0.4175061285495758, 0.4531165659427643, 0.7083004713058472], "cos_apical_negg": [0.09154856950044632, 0.15246111154556274, 0.36129021644592285], "cos_Ac_negg": [0.41741645336151123, 0.45313185453414917, 0.708194375038147], "r_norm": [1.053849458694458, 1.0565844774246216, 1.060617446899414]}, {"step": 3200, "epoch": 6, "train_loss": 0.09819801151752472, "cos_r_negg": [0.07863219082355499, 0.14805670082569122, 0.35639455914497375], "cos_innovation_negg": [0.4197821021080017, 0.4336961507797241, 0.6829315423965454], "cos_apical_negg": [0.07863219082355499, 0.14805670082569122, 0.35639455914497375], "cos_Ac_negg": [0.4195158779621124, 0.4333699643611908, 0.6826649904251099], "r_norm": [1.0234484672546387, 1.0238585472106934, 1.0276715755462646]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9607, "test_loss": 0.16267271842956543}, {"step": 3300, "epoch": 7, "train_loss": 0.028529947623610497, "cos_r_negg": [0.08966170251369476, 0.12818852066993713, 0.34817975759506226], "cos_innovation_negg": [0.441426545381546, 0.4613744616508484, 0.6785696744918823], "cos_apical_negg": [0.08966170251369476, 0.12818852066993713, 0.34817975759506226], "cos_Ac_negg": [0.4409579634666443, 0.4609857499599457, 0.6791725754737854], "r_norm": [1.0220474004745483, 1.0210413932800293, 1.025068759918213]}, {"step": 3400, "epoch": 7, "train_loss": 0.1324930042028427, "cos_r_negg": [0.07912766933441162, 0.131761372089386, 0.3849921226501465], "cos_innovation_negg": [0.4406862258911133, 0.4612444043159485, 0.6928120851516724], "cos_apical_negg": [0.07912766933441162, 0.131761372089386, 0.3849921226501465], "cos_Ac_negg": [0.4403899908065796, 0.46111372113227844, 0.6927999258041382], "r_norm": [1.0272427797317505, 1.0293800830841064, 1.0310218334197998]}, {"step": 3500, "epoch": 7, "train_loss": 0.05119926109910011, "cos_r_negg": [0.07785937935113907, 0.13771826028823853, 0.31989145278930664], "cos_innovation_negg": [0.39601752161979675, 0.4081972539424896, 0.6666200160980225], "cos_apical_negg": [0.07785937935113907, 0.13771826028823853, 0.31989145278930664], "cos_Ac_negg": [0.3957563042640686, 0.4080226421356201, 0.6665476560592651], "r_norm": [1.0206598043441772, 1.022580623626709, 1.0244653224945068], "ldr": {"dL_sdil": -0.0055401623249053955, "dL_bp": -0.006562232971191406, "ratio": 0.844249564016858}}, {"step": 3600, "epoch": 7, "train_loss": 0.009449993260204792, "cos_r_negg": [0.09007138013839722, 0.09390336275100708, 0.364920049905777], "cos_innovation_negg": [0.4198237955570221, 0.42908626794815063, 0.6880031228065491], "cos_apical_negg": [0.09007138013839722, 0.09390336275100708, 0.364920049905777], "cos_Ac_negg": [0.41977348923683167, 0.4300592541694641, 0.6880263686180115], "r_norm": [0.99738609790802, 0.9973214864730835, 1.0014511346817017]}, {"step": 3700, "epoch": 7, "train_loss": 0.047529637813568115, "cos_r_negg": [0.060060493648052216, 0.07941828668117523, 0.26429396867752075], "cos_innovation_negg": [0.43672502040863037, 0.4856759309768677, 0.7090200185775757], "cos_apical_negg": [0.060060493648052216, 0.07941828668117523, 0.26429396867752075], "cos_Ac_negg": [0.43659698963165283, 0.4856643080711365, 0.7091152667999268], "r_norm": [1.0114314556121826, 1.0112817287445068, 1.013547658920288]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9619, "test_loss": 0.1646844841003418}, {"step": 3800, "epoch": 8, "train_loss": 0.028165673837065697, "cos_r_negg": [0.04935648664832115, 0.07586020231246948, 0.29833394289016724], "cos_innovation_negg": [0.39825111627578735, 0.44790321588516235, 0.6613509654998779], "cos_apical_negg": [0.04935648664832115, 0.07586020231246948, 0.29833394289016724], "cos_Ac_negg": [0.39816486835479736, 0.4477677643299103, 0.6615231037139893], "r_norm": [1.0284254550933838, 1.0292689800262451, 1.0327450037002563]}, {"step": 3900, "epoch": 8, "train_loss": 0.1413274109363556, "cos_r_negg": [0.057543277740478516, 0.1653311848640442, 0.3022421598434448], "cos_innovation_negg": [0.3702201247215271, 0.364401638507843, 0.6903688311576843], "cos_apical_negg": [0.057543277740478516, 0.1653311848640442, 0.3022421598434448], "cos_Ac_negg": [0.36931949853897095, 0.3637664020061493, 0.6898601055145264], "r_norm": [1.0219491720199585, 1.0218803882598877, 1.0267157554626465]}, {"step": 4000, "epoch": 8, "train_loss": 0.037294983863830566, "cos_r_negg": [0.04054345190525055, 0.09202315658330917, 0.3343167304992676], "cos_innovation_negg": [0.3908749520778656, 0.4211050868034363, 0.6868146657943726], "cos_apical_negg": [0.04054345190525055, 0.09202315658330917, 0.3343167304992676], "cos_Ac_negg": [0.3908200263977051, 0.42087793350219727, 0.6864671111106873], "r_norm": [1.0049583911895752, 1.005995273590088, 1.0071860551834106], "ldr": {"dL_sdil": -0.005113609135150909, "dL_bp": -0.006626486778259277, "ratio": 0.7716923471315236}}, {"step": 4100, "epoch": 8, "train_loss": 0.024643462151288986, "cos_r_negg": [0.05470426380634308, 0.13417969644069672, 0.33288973569869995], "cos_innovation_negg": [0.36566293239593506, 0.3673812747001648, 0.7012430429458618], "cos_apical_negg": [0.05470426380634308, 0.13417969644069672, 0.33288973569869995], "cos_Ac_negg": [0.36525222659111023, 0.3670940399169922, 0.701558530330658], "r_norm": [0.9790952205657959, 0.9794970154762268, 0.979072093963623]}, {"step": 4200, "epoch": 8, "train_loss": 0.11996988952159882, "cos_r_negg": [0.0717785581946373, 0.12417430430650711, 0.3153742849826813], "cos_innovation_negg": [0.3397974669933319, 0.3639867603778839, 0.7073224782943726], "cos_apical_negg": [0.0717785581946373, 0.12417430430650711, 0.3153742849826813], "cos_Ac_negg": [0.3396352529525757, 0.3636566400527954, 0.708104133605957], "r_norm": [0.9776666760444641, 0.977496325969696, 0.9800012111663818]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.962, "test_loss": 0.1710110767364502}, {"step": 4300, "epoch": 9, "train_loss": 0.03403007239103317, "cos_r_negg": [0.05581548810005188, 0.13285601139068604, 0.3589896559715271], "cos_innovation_negg": [0.35469383001327515, 0.3764450252056122, 0.7123684883117676], "cos_apical_negg": [0.05581548810005188, 0.13285601139068604, 0.3589896559715271], "cos_Ac_negg": [0.3544830083847046, 0.3760649561882019, 0.7125866413116455], "r_norm": [0.97840416431427, 0.9785627126693726, 0.9797685742378235]}, {"step": 4400, "epoch": 9, "train_loss": 0.03238349407911301, "cos_r_negg": [0.07354895770549774, 0.15934056043624878, 0.36559784412384033], "cos_innovation_negg": [0.40034839510917664, 0.42497092485427856, 0.7239202857017517], "cos_apical_negg": [0.07354895770549774, 0.15934056043624878, 0.36559784412384033], "cos_Ac_negg": [0.40018779039382935, 0.4246886968612671, 0.7229862213134766], "r_norm": [1.008237361907959, 1.0079395771026611, 1.0122385025024414]}, {"step": 4500, "epoch": 9, "train_loss": 0.1467689424753189, "cos_r_negg": [0.06329061836004257, 0.15482372045516968, 0.34820833802223206], "cos_innovation_negg": [0.3648613691329956, 0.3886018991470337, 0.7092785835266113], "cos_apical_negg": [0.06329061836004257, 0.15482372045516968, 0.34820833802223206], "cos_Ac_negg": [0.36520254611968994, 0.38881176710128784, 0.7101279497146606], "r_norm": [1.0042691230773926, 1.0046032667160034, 1.005810260772705], "ldr": {"dL_sdil": -0.005316473543643951, "dL_bp": -0.007558912038803101, "ratio": 0.7033384588089183}}, {"step": 4600, "epoch": 9, "train_loss": 0.012906771153211594, "cos_r_negg": [0.06826816499233246, 0.11718006432056427, 0.3150479793548584], "cos_innovation_negg": [0.3665471374988556, 0.40594732761383057, 0.7223443388938904], "cos_apical_negg": [0.06826816499233246, 0.11718006432056427, 0.3150479793548584], "cos_Ac_negg": [0.36649179458618164, 0.40595564246177673, 0.7226409316062927], "r_norm": [1.0136744976043701, 1.0134626626968384, 1.0152064561843872]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9637, "test_loss": 0.16485629844665528}, {"step": 4700, "epoch": 10, "train_loss": 0.02736743539571762, "cos_r_negg": [0.03550593927502632, 0.11548344790935516, 0.28253379464149475], "cos_innovation_negg": [0.3464192748069763, 0.3614247143268585, 0.693299412727356], "cos_apical_negg": [0.03550593927502632, 0.11548344790935516, 0.28253379464149475], "cos_Ac_negg": [0.3461849093437195, 0.3606976866722107, 0.693226158618927], "r_norm": [1.0213532447814941, 1.020444631576538, 1.0231938362121582]}, {"step": 4800, "epoch": 10, "train_loss": 0.0703713521361351, "cos_r_negg": [0.044721659272909164, 0.07623544335365295, 0.242234468460083], "cos_innovation_negg": [0.35500195622444153, 0.38033363223075867, 0.7140325307846069], "cos_apical_negg": [0.044721659272909164, 0.07623544335365295, 0.242234468460083], "cos_Ac_negg": [0.35475096106529236, 0.3799006938934326, 0.7137417793273926], "r_norm": [0.9935460090637207, 0.9923553466796875, 0.9948216676712036]}, {"step": 4900, "epoch": 10, "train_loss": 0.02629293128848076, "cos_r_negg": [0.06685224920511246, 0.13649609684944153, 0.2876155376434326], "cos_innovation_negg": [0.3617466688156128, 0.4214091897010803, 0.7168684005737305], "cos_apical_negg": [0.06685224920511246, 0.13649609684944153, 0.2876155376434326], "cos_Ac_negg": [0.3615431785583496, 0.4209834933280945, 0.7167174816131592], "r_norm": [1.0347400903701782, 1.0337830781936646, 1.0375102758407593]}, {"step": 5000, "epoch": 10, "train_loss": 0.035900868475437164, "cos_r_negg": [0.025756340473890305, 0.13357065618038177, 0.3234238624572754], "cos_innovation_negg": [0.3522346019744873, 0.40994662046432495, 0.6928673982620239], "cos_apical_negg": [0.025756340473890305, 0.13357065618038177, 0.3234238624572754], "cos_Ac_negg": [0.35224539041519165, 0.40978020429611206, 0.6926594972610474], "r_norm": [1.0103343725204468, 1.0091902017593384, 1.010603904724121], "ldr": {"dL_sdil": -0.005174554884433746, "dL_bp": -0.0073034390807151794, "ratio": 0.7085093511763824}}, {"step": 5100, "epoch": 10, "train_loss": 0.05992225557565689, "cos_r_negg": [0.04124075174331665, 0.10998278856277466, 0.27149146795272827], "cos_innovation_negg": [0.38109320402145386, 0.4332895576953888, 0.708886981010437], "cos_apical_negg": [0.04124075174331665, 0.10998278856277466, 0.27149146795272827], "cos_Ac_negg": [0.38090264797210693, 0.43301063776016235, 0.7095015048980713], "r_norm": [0.9967202544212341, 0.997312605381012, 0.997156023979187]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9631, "test_loss": 0.16887755584716796}, {"step": 5200, "epoch": 11, "train_loss": 0.029580220580101013, "cos_r_negg": [0.027782924473285675, 0.11598636955022812, 0.2720359265804291], "cos_innovation_negg": [0.32047146558761597, 0.35808008909225464, 0.6741222143173218], "cos_apical_negg": [0.027782924473285675, 0.11598636955022812, 0.2720359265804291], "cos_Ac_negg": [0.32039231061935425, 0.35767117142677307, 0.6737675070762634], "r_norm": [1.0095304250717163, 1.0074961185455322, 1.012148380279541]}, {"step": 5300, "epoch": 11, "train_loss": 0.022948311641812325, "cos_r_negg": [0.034091852605342865, 0.06341452896595001, 0.21252983808517456], "cos_innovation_negg": [0.36609888076782227, 0.42277854681015015, 0.7314366102218628], "cos_apical_negg": [0.034091852605342865, 0.06341452896595001, 0.21252983808517456], "cos_Ac_negg": [0.3660215437412262, 0.4229591190814972, 0.7318074107170105], "r_norm": [0.9903703927993774, 0.9895915985107422, 0.9912302494049072]}, {"step": 5400, "epoch": 11, "train_loss": 0.09156230092048645, "cos_r_negg": [0.024518117308616638, 0.07620741426944733, 0.2544670104980469], "cos_innovation_negg": [0.3567865490913391, 0.41314697265625, 0.6894890069961548], "cos_apical_negg": [0.024518117308616638, 0.07620741426944733, 0.2544670104980469], "cos_Ac_negg": [0.35719090700149536, 0.4126911163330078, 0.6897022128105164], "r_norm": [0.9840506315231323, 0.9835418462753296, 0.984965443611145]}, {"step": 5500, "epoch": 11, "train_loss": 0.06469833850860596, "cos_r_negg": [0.028588611632585526, 0.12334965914487839, 0.24590280652046204], "cos_innovation_negg": [0.334442138671875, 0.33600616455078125, 0.7357508540153503], "cos_apical_negg": [0.028588611632585526, 0.12334965914487839, 0.24590280652046204], "cos_Ac_negg": [0.3342592418193817, 0.33551597595214844, 0.7359646558761597], "r_norm": [1.0026648044586182, 1.0012426376342773, 1.0037378072738647], "ldr": {"dL_sdil": -0.005138382315635681, "dL_bp": -0.007787100970745087, "ratio": 0.6598581853426294}}, {"step": 5600, "epoch": 11, "train_loss": 0.01619894802570343, "cos_r_negg": [0.02412668615579605, 0.06913219392299652, 0.25795450806617737], "cos_innovation_negg": [0.3788745403289795, 0.4205939769744873, 0.7200464010238647], "cos_apical_negg": [0.02412668615579605, 0.06913219392299652, 0.25795450806617737], "cos_Ac_negg": [0.377731591463089, 0.42007142305374146, 0.7202447652816772], "r_norm": [1.010632872581482, 1.00955069065094, 1.0115344524383545]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9638, "test_loss": 0.17477376174926756}, {"step": 5700, "epoch": 12, "train_loss": 0.034003935754299164, "cos_r_negg": [0.02326776459813118, 0.09680469334125519, 0.2676955461502075], "cos_innovation_negg": [0.332442969083786, 0.40938708186149597, 0.7143099308013916], "cos_apical_negg": [0.02326776459813118, 0.09680469334125519, 0.2676955461502075], "cos_Ac_negg": [0.33246880769729614, 0.4096144437789917, 0.7147245407104492], "r_norm": [0.9652283191680908, 0.9628087878227234, 0.9655753374099731]}, {"step": 5800, "epoch": 12, "train_loss": 0.011000153608620167, "cos_r_negg": [0.003949150908738375, 0.11836135387420654, 0.2192959487438202], "cos_innovation_negg": [0.35420942306518555, 0.44399017095565796, 0.6782984733581543], "cos_apical_negg": [0.003949150908738375, 0.11836135387420654, 0.2192959487438202], "cos_Ac_negg": [0.3537759482860565, 0.4441183805465698, 0.6786057949066162], "r_norm": [1.0089538097381592, 1.0071743726730347, 1.0095679759979248]}, {"step": 5900, "epoch": 12, "train_loss": 0.04633845388889313, "cos_r_negg": [0.02327817864716053, 0.03072489984333515, 0.2638982832431793], "cos_innovation_negg": [0.33669018745422363, 0.3953905403614044, 0.6866704821586609], "cos_apical_negg": [0.02327817864716053, 0.03072489984333515, 0.2638982832431793], "cos_Ac_negg": [0.3359820246696472, 0.3951113820075989, 0.6870233416557312], "r_norm": [1.0110232830047607, 1.0101172924041748, 1.0133225917816162]}, {"step": 6000, "epoch": 12, "train_loss": 0.01362815871834755, "cos_r_negg": [0.03887077793478966, 0.11401763558387756, 0.25929033756256104], "cos_innovation_negg": [0.330759197473526, 0.3829606771469116, 0.7363024353981018], "cos_apical_negg": [0.03887077793478966, 0.11401763558387756, 0.25929033756256104], "cos_Ac_negg": [0.3305964469909668, 0.38287049531936646, 0.7362428307533264], "r_norm": [0.994838297367096, 0.9944480061531067, 0.9945006966590881], "ldr": {"dL_sdil": -0.004394859075546265, "dL_bp": -0.006933242082595825, "ratio": 0.6338822477551248}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9646, "test_loss": 0.16979231948852538}, {"step": 6100, "epoch": 13, "train_loss": 0.06225595250725746, "cos_r_negg": [0.010341176763176918, 0.15007269382476807, 0.26343023777008057], "cos_innovation_negg": [0.3087495267391205, 0.33682119846343994, 0.7710036039352417], "cos_apical_negg": [0.010341176763176918, 0.15007269382476807, 0.26343023777008057], "cos_Ac_negg": [0.309277206659317, 0.3375481367111206, 0.7723146080970764], "r_norm": [0.992320716381073, 0.9911161065101624, 0.9919863939285278]}, {"step": 6200, "epoch": 13, "train_loss": 0.0032057627104222775, "cos_r_negg": [0.021069344133138657, 0.12110094726085663, 0.24954718351364136], "cos_innovation_negg": [0.32438939809799194, 0.37345829606056213, 0.7641387581825256], "cos_apical_negg": [0.021069344133138657, 0.12110094726085663, 0.24954718351364136], "cos_Ac_negg": [0.3244231641292572, 0.37292492389678955, 0.7640959620475769], "r_norm": [0.9851340055465698, 0.9834593534469604, 0.9847379922866821]}, {"step": 6300, "epoch": 13, "train_loss": 0.008786734193563461, "cos_r_negg": [0.026174163445830345, 0.14485730230808258, 0.20660057663917542], "cos_innovation_negg": [0.30421939492225647, 0.34772437810897827, 0.7134988307952881], "cos_apical_negg": [0.026174163445830345, 0.14485730230808258, 0.20660057663917542], "cos_Ac_negg": [0.30417656898498535, 0.3474142253398895, 0.7138352394104004], "r_norm": [0.9829367995262146, 0.9817464351654053, 0.9827190041542053]}, {"step": 6400, "epoch": 13, "train_loss": 0.029361091554164886, "cos_r_negg": [0.018825875595211983, 0.06510050594806671, 0.20756199955940247], "cos_innovation_negg": [0.32970887422561646, 0.3689943552017212, 0.7224449515342712], "cos_apical_negg": [0.018825875595211983, 0.06510050594806671, 0.20756199955940247], "cos_Ac_negg": [0.3299565017223358, 0.3688269853591919, 0.7231218814849854], "r_norm": [1.0003130435943604, 0.998401403427124, 1.001128911972046]}, {"step": 6500, "epoch": 13, "train_loss": 0.008670948445796967, "cos_r_negg": [0.007858971133828163, 0.12014691531658173, 0.22846658527851105], "cos_innovation_negg": [0.3156766891479492, 0.3323468267917633, 0.7432212829589844], "cos_apical_negg": [0.007858971133828163, 0.12014691531658173, 0.22846658527851105], "cos_Ac_negg": [0.31595155596733093, 0.3322823941707611, 0.7441815137863159], "r_norm": [0.980766773223877, 0.9787405729293823, 0.9799807071685791], "ldr": {"dL_sdil": -0.004171811044216156, "dL_bp": -0.007981650531291962, "ratio": 0.5226752321290719}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9642, "test_loss": 0.1732793312072754}, {"step": 6600, "epoch": 14, "train_loss": 0.008569706231355667, "cos_r_negg": [-0.004409132990986109, 0.10667832940816879, 0.2484140694141388], "cos_innovation_negg": [0.29996323585510254, 0.33741700649261475, 0.714909017086029], "cos_apical_negg": [-0.004409132990986109, 0.10667832940816879, 0.2484140694141388], "cos_Ac_negg": [0.29964345693588257, 0.33711785078048706, 0.7159640192985535], "r_norm": [0.9837241172790527, 0.9821308851242065, 0.9836736917495728]}, {"step": 6700, "epoch": 14, "train_loss": 0.07513140887022018, "cos_r_negg": [0.0029986146837472916, 0.12318495661020279, 0.21719655394554138], "cos_innovation_negg": [0.28236839175224304, 0.3176575303077698, 0.7294604778289795], "cos_apical_negg": [0.0029986146837472916, 0.12318495661020279, 0.21719655394554138], "cos_Ac_negg": [0.28175628185272217, 0.3161846697330475, 0.7299823760986328], "r_norm": [1.0008494853973389, 0.9995094537734985, 1.0006990432739258]}, {"step": 6800, "epoch": 14, "train_loss": 0.06361684203147888, "cos_r_negg": [0.02827482670545578, 0.03142590448260307, 0.26733407378196716], "cos_innovation_negg": [0.3290252089500427, 0.398573100566864, 0.773834228515625], "cos_apical_negg": [0.02827482670545578, 0.03142590448260307, 0.26733407378196716], "cos_Ac_negg": [0.329598993062973, 0.399262011051178, 0.774982213973999], "r_norm": [0.9957860708236694, 0.9938015937805176, 0.9954384565353394]}, {"step": 6900, "epoch": 14, "train_loss": 0.009432310238480568, "cos_r_negg": [0.005537485703825951, 0.10541363060474396, 0.22380328178405762], "cos_innovation_negg": [0.33142176270484924, 0.35452109575271606, 0.7206307053565979], "cos_apical_negg": [0.005537485703825951, 0.10541363060474396, 0.22380328178405762], "cos_Ac_negg": [0.33240050077438354, 0.354592502117157, 0.7211755514144897], "r_norm": [0.9861862063407898, 0.9840484857559204, 0.9847438931465149]}, {"step": 7000, "epoch": 14, "train_loss": 0.055023930966854095, "cos_r_negg": [0.021639054641127586, 0.11997617781162262, 0.2204824537038803], "cos_innovation_negg": [0.30473893880844116, 0.3262280225753784, 0.7341196537017822], "cos_apical_negg": [0.021639054641127586, 0.11997617781162262, 0.2204824537038803], "cos_Ac_negg": [0.30584344267845154, 0.32628151774406433, 0.7359507083892822], "r_norm": [0.9695146083831787, 0.9673624038696289, 0.9677636623382568], "ldr": {"dL_sdil": -0.00318213552236557, "dL_bp": -0.0056747570633888245, "ratio": 0.560752731230626}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9642, "test_loss": 0.18486077995300293}], "final": {"test_acc": 0.9642, "test_loss": 0.18486077995300293, "wall_s": 34.16561818122864, "cos_r_negg": [0.020560957491397858, 0.10119639337062836, 0.23658712208271027], "cos_innovation_negg": [0.30665135383605957, 0.3345761001110077, 0.7279347777366638], "cos_apical_negg": [0.020560957491397858, 0.10119639337062836, 0.23658712208271027], "cos_Ac_negg": [0.30704784393310547, 0.3338749408721924, 0.7279426455497742]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_residual_s0.json b/results/nuis_ctrl_v1_rho0p05_residual_s0.json new file mode 100644 index 0000000..3c0a6ac --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_residual_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_residual_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.01952279359102249, -0.025533504784107208, 0.033745452761650085], "cos_innovation_negg": [-0.01952279359102249, -0.025533504784107208, 0.033745452761650085], "cos_apical_negg": [-0.019518455490469933, -0.025709278881549835, 0.03309042379260063], "cos_Ac_negg": [-0.01952274888753891, -0.02553350292146206, 0.03374544531106949], "r_norm": [4.898251056671143, 4.826769828796387, 4.823906898498535], "ldr": {"dL_sdil": -0.009195327758789062, "dL_bp": -0.2406785488128662, "ratio": 0.03820584677838766}}, {"step": 100, "epoch": 0, "train_loss": 0.20553797483444214, "cos_r_negg": [0.4579622447490692, 0.38216572999954224, 0.3608860969543457], "cos_innovation_negg": [0.4579622447490692, 0.38216572999954224, 0.3608860969543457], "cos_apical_negg": [0.19300329685211182, 0.17585119605064392, 0.21503859758377075], "cos_Ac_negg": [0.4582347273826599, 0.3824673295021057, 0.3610897660255432], "r_norm": [0.8250458240509033, 0.8087376356124878, 0.8063828945159912]}, {"step": 200, "epoch": 0, "train_loss": 0.3347880244255066, "cos_r_negg": [0.484967976808548, 0.40824341773986816, 0.3901892900466919], "cos_innovation_negg": [0.484967976808548, 0.40824341773986816, 0.3901892900466919], "cos_apical_negg": [0.18476158380508423, 0.17060211300849915, 0.2136172205209732], "cos_Ac_negg": [0.48558053374290466, 0.40904223918914795, 0.39106833934783936], "r_norm": [0.739167332649231, 0.7221766710281372, 0.7204556465148926]}, {"step": 300, "epoch": 0, "train_loss": 0.1608467698097229, "cos_r_negg": [0.5106900334358215, 0.4493744671344757, 0.4224192500114441], "cos_innovation_negg": [0.5106900334358215, 0.4493744671344757, 0.4224192500114441], "cos_apical_negg": [0.1733049750328064, 0.1539825201034546, 0.1863439977169037], "cos_Ac_negg": [0.5108871459960938, 0.44945770502090454, 0.42250585556030273], "r_norm": [0.5548641681671143, 0.5450409054756165, 0.5431962013244629]}, {"step": 400, "epoch": 0, "train_loss": 0.18942078948020935, "cos_r_negg": [0.5131446123123169, 0.4551883339881897, 0.42917412519454956], "cos_innovation_negg": [0.5131446123123169, 0.4551883339881897, 0.42917412519454956], "cos_apical_negg": [0.15334802865982056, 0.15426218509674072, 0.18405623733997345], "cos_Ac_negg": [0.5133811235427856, 0.4552389979362488, 0.4292002320289612], "r_norm": [0.5758621096611023, 0.5641695857048035, 0.5628612041473389]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9464, "test_loss": 0.1857574264526367}, {"step": 500, "epoch": 1, "train_loss": 0.2046205848455429, "cos_r_negg": [0.523687481880188, 0.4869231581687927, 0.4766062796115875], "cos_innovation_negg": [0.523687481880188, 0.4869231581687927, 0.4766062796115875], "cos_apical_negg": [0.13296380639076233, 0.1439264565706253, 0.18738295137882233], "cos_Ac_negg": [0.5244132876396179, 0.4869997501373291, 0.4767279326915741], "r_norm": [0.5014256238937378, 0.4845983386039734, 0.4856627881526947], "ldr": {"dL_sdil": -0.012597888708114624, "dL_bp": -0.010928422212600708, "ratio": 1.1527637259099475}}, {"step": 600, "epoch": 1, "train_loss": 0.08640950918197632, "cos_r_negg": [0.5135207176208496, 0.48049765825271606, 0.4631809592247009], "cos_innovation_negg": [0.5135207176208496, 0.48049765825271606, 0.4631809592247009], "cos_apical_negg": [0.13356101512908936, 0.13810202479362488, 0.18871255218982697], "cos_Ac_negg": [0.5141767859458923, 0.48077037930488586, 0.46318358182907104], "r_norm": [0.4284065067768097, 0.4179823696613312, 0.4188630282878876]}, {"step": 700, "epoch": 1, "train_loss": 0.22193440794944763, "cos_r_negg": [0.5253515839576721, 0.49967077374458313, 0.47384780645370483], "cos_innovation_negg": [0.5253515839576721, 0.49967077374458313, 0.47384780645370483], "cos_apical_negg": [0.13406303524971008, 0.13647781312465668, 0.20093558728694916], "cos_Ac_negg": [0.5262846350669861, 0.49967145919799805, 0.47360819578170776], "r_norm": [0.4450308084487915, 0.4301424026489258, 0.43267372250556946]}, {"step": 800, "epoch": 1, "train_loss": 0.16923829913139343, "cos_r_negg": [0.5265134572982788, 0.5200668573379517, 0.48621031641960144], "cos_innovation_negg": [0.5265134572982788, 0.5200668573379517, 0.48621031641960144], "cos_apical_negg": [0.1261288970708847, 0.12328580021858215, 0.1760111153125763], "cos_Ac_negg": [0.5271130800247192, 0.5202675461769104, 0.4861285984516144], "r_norm": [0.3633059561252594, 0.35162240266799927, 0.35249751806259155]}, {"step": 900, "epoch": 1, "train_loss": 0.2054840624332428, "cos_r_negg": [0.5366386771202087, 0.5371055603027344, 0.4891264736652374], "cos_innovation_negg": [0.5366386771202087, 0.5371055603027344, 0.4891264736652374], "cos_apical_negg": [0.12613967061042786, 0.1435614675283432, 0.17565500736236572], "cos_Ac_negg": [0.536929726600647, 0.5370739102363586, 0.4889616072177887], "r_norm": [0.3984663486480713, 0.3892437219619751, 0.38930922746658325]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9569, "test_loss": 0.14992256050109865}, {"step": 1000, "epoch": 2, "train_loss": 0.03922855854034424, "cos_r_negg": [0.5354604721069336, 0.5387663841247559, 0.49675408005714417], "cos_innovation_negg": [0.5354604721069336, 0.5387663841247559, 0.49675408005714417], "cos_apical_negg": [0.10201052576303482, 0.11773139238357544, 0.16800633072853088], "cos_Ac_negg": [0.5359271764755249, 0.538806140422821, 0.49607735872268677], "r_norm": [0.33832865953445435, 0.32900702953338623, 0.3283848166465759], "ldr": {"dL_sdil": -0.00833958387374878, "dL_bp": -0.0074491947889328, "ratio": 1.1195282322511988}}, {"step": 1100, "epoch": 2, "train_loss": 0.1274726837873459, "cos_r_negg": [0.5607089400291443, 0.5610537528991699, 0.5137262344360352], "cos_innovation_negg": [0.5607089400291443, 0.5610537528991699, 0.5137262344360352], "cos_apical_negg": [0.10683763772249222, 0.12646016478538513, 0.16395971179008484], "cos_Ac_negg": [0.5610625743865967, 0.561252236366272, 0.5135065913200378], "r_norm": [0.3592675030231476, 0.34692731499671936, 0.3486380875110626]}, {"step": 1200, "epoch": 2, "train_loss": 0.15907587110996246, "cos_r_negg": [0.5624092817306519, 0.5603923797607422, 0.514489471912384], "cos_innovation_negg": [0.5624092817306519, 0.5603923797607422, 0.514489471912384], "cos_apical_negg": [0.10560047626495361, 0.1084093376994133, 0.14780128002166748], "cos_Ac_negg": [0.5627895593643188, 0.5604621767997742, 0.5141286253929138], "r_norm": [0.32979658246040344, 0.3189634084701538, 0.3202262818813324]}, {"step": 1300, "epoch": 2, "train_loss": 0.12210610508918762, "cos_r_negg": [0.5728862285614014, 0.5762075185775757, 0.532311201095581], "cos_innovation_negg": [0.5728862285614014, 0.5762075185775757, 0.532311201095581], "cos_apical_negg": [0.09865699708461761, 0.11101168394088745, 0.15804867446422577], "cos_Ac_negg": [0.573574423789978, 0.5762590765953064, 0.5317193269729614], "r_norm": [0.3354035019874573, 0.3264496326446533, 0.32440072298049927]}, {"step": 1400, "epoch": 2, "train_loss": 0.16060467064380646, "cos_r_negg": [0.5716283321380615, 0.577843427658081, 0.5309174060821533], "cos_innovation_negg": [0.5716283321380615, 0.577843427658081, 0.5309174060821533], "cos_apical_negg": [0.10109320282936096, 0.12631100416183472, 0.16680295765399933], "cos_Ac_negg": [0.5754722952842712, 0.5790044069290161, 0.5308045744895935], "r_norm": [0.3186105191707611, 0.3082095980644226, 0.3089616894721985]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9655, "test_loss": 0.12051229820251465}, {"step": 1500, "epoch": 3, "train_loss": 0.06386608630418777, "cos_r_negg": [0.5805710554122925, 0.5955780744552612, 0.5463205575942993], "cos_innovation_negg": [0.5805710554122925, 0.5955780744552612, 0.5463205575942993], "cos_apical_negg": [0.07950741797685623, 0.10170567780733109, 0.1538735181093216], "cos_Ac_negg": [0.581992506980896, 0.5966823101043701, 0.5457150936126709], "r_norm": [0.2611697316169739, 0.2530985474586487, 0.2531789541244507], "ldr": {"dL_sdil": -0.00545801967382431, "dL_bp": -0.005244113504886627, "ratio": 1.0407897671814996}}, {"step": 1600, "epoch": 3, "train_loss": 0.06963219493627548, "cos_r_negg": [0.5691196918487549, 0.5914679765701294, 0.5502804517745972], "cos_innovation_negg": [0.5691196918487549, 0.5914679765701294, 0.5502804517745972], "cos_apical_negg": [0.08091358840465546, 0.10967737436294556, 0.1509772539138794], "cos_Ac_negg": [0.571120023727417, 0.5928018093109131, 0.5498823523521423], "r_norm": [0.2819720506668091, 0.2724171280860901, 0.27314233779907227]}, {"step": 1700, "epoch": 3, "train_loss": 0.06539039313793182, "cos_r_negg": [0.5813878774642944, 0.5956884026527405, 0.547366738319397], "cos_innovation_negg": [0.5813878774642944, 0.5956884026527405, 0.547366738319397], "cos_apical_negg": [0.0911014974117279, 0.10569155216217041, 0.15141209959983826], "cos_Ac_negg": [0.5834836959838867, 0.5967861413955688, 0.5467633008956909], "r_norm": [0.25422245264053345, 0.24650141596794128, 0.24772970378398895]}, {"step": 1800, "epoch": 3, "train_loss": 0.016470560804009438, "cos_r_negg": [0.5626012086868286, 0.5824457406997681, 0.5532260537147522], "cos_innovation_negg": [0.5626012086868286, 0.5824457406997681, 0.5532260537147522], "cos_apical_negg": [0.0819510817527771, 0.09958356618881226, 0.1566336750984192], "cos_Ac_negg": [0.568524956703186, 0.5860669612884521, 0.5519437789916992], "r_norm": [0.248224139213562, 0.2410775125026703, 0.24029222130775452]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9652, "test_loss": 0.12335003890991211}, {"step": 1900, "epoch": 4, "train_loss": 0.026236163452267647, "cos_r_negg": [0.5891915559768677, 0.6053824424743652, 0.5565018057823181], "cos_innovation_negg": [0.5891915559768677, 0.6053824424743652, 0.5565018057823181], "cos_apical_negg": [0.09248057007789612, 0.1169896274805069, 0.16277912259101868], "cos_Ac_negg": [0.5900899171829224, 0.6065192222595215, 0.5559866428375244], "r_norm": [0.2781800627708435, 0.27161142230033875, 0.27034926414489746]}, {"step": 2000, "epoch": 4, "train_loss": 0.057471826672554016, "cos_r_negg": [0.569402813911438, 0.5875678062438965, 0.5663565397262573], "cos_innovation_negg": [0.569402813911438, 0.5875678062438965, 0.5663565397262573], "cos_apical_negg": [0.052954062819480896, 0.0836525708436966, 0.13660719990730286], "cos_Ac_negg": [0.5739847421646118, 0.5923200845718384, 0.5661741495132446], "r_norm": [0.21866536140441895, 0.21067006886005402, 0.21089935302734375], "ldr": {"dL_sdil": -0.004901431500911713, "dL_bp": -0.005205012857913971, "ratio": 0.9416751955683111}}, {"step": 2100, "epoch": 4, "train_loss": 0.018851609900593758, "cos_r_negg": [0.5779240131378174, 0.5937851667404175, 0.5687053799629211], "cos_innovation_negg": [0.5779240131378174, 0.5937851667404175, 0.5687053799629211], "cos_apical_negg": [0.060382746160030365, 0.09317074716091156, 0.14972719550132751], "cos_Ac_negg": [0.5829090476036072, 0.596392035484314, 0.5674302577972412], "r_norm": [0.25009214878082275, 0.24291518330574036, 0.24175116419792175]}, {"step": 2200, "epoch": 4, "train_loss": 0.02432655729353428, "cos_r_negg": [0.5698356628417969, 0.5986080169677734, 0.5691801905632019], "cos_innovation_negg": [0.5698356628417969, 0.5986080169677734, 0.5691801905632019], "cos_apical_negg": [0.04793553054332733, 0.09527935087680817, 0.14640530943870544], "cos_Ac_negg": [0.5759973526000977, 0.6028170585632324, 0.5682347416877747], "r_norm": [0.22581236064434052, 0.2182098776102066, 0.2178492397069931]}, {"step": 2300, "epoch": 4, "train_loss": 0.1269402801990509, "cos_r_negg": [0.5726011991500854, 0.5903053283691406, 0.5651850700378418], "cos_innovation_negg": [0.5726011991500854, 0.5903053283691406, 0.5651850700378418], "cos_apical_negg": [0.07685291767120361, 0.09186537563800812, 0.13838668167591095], "cos_Ac_negg": [0.5830402970314026, 0.6027272939682007, 0.5688943862915039], "r_norm": [0.2527035176753998, 0.24287879467010498, 0.2436361163854599]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9693, "test_loss": 0.10827527141571044}, {"step": 2400, "epoch": 5, "train_loss": 0.08427467942237854, "cos_r_negg": [0.5815878510475159, 0.5953119993209839, 0.5846785306930542], "cos_innovation_negg": [0.5815878510475159, 0.5953119993209839, 0.5846785306930542], "cos_apical_negg": [0.07297449558973312, 0.08918994665145874, 0.1430235356092453], "cos_Ac_negg": [0.5873320698738098, 0.6000885963439941, 0.5838549137115479], "r_norm": [0.21910353004932404, 0.21354863047599792, 0.21174027025699615]}, {"step": 2500, "epoch": 5, "train_loss": 0.03700476884841919, "cos_r_negg": [0.5740196704864502, 0.5969030857086182, 0.5744849443435669], "cos_innovation_negg": [0.5740196704864502, 0.5969030857086182, 0.5744849443435669], "cos_apical_negg": [0.04668359458446503, 0.08081283420324326, 0.1406819075345993], "cos_Ac_negg": [0.5860415101051331, 0.6097989082336426, 0.5790238380432129], "r_norm": [0.21384358406066895, 0.20827405154705048, 0.20768722891807556], "ldr": {"dL_sdil": -0.005959019064903259, "dL_bp": -0.007187977433204651, "ratio": 0.8290258449304175}}, {"step": 2600, "epoch": 5, "train_loss": 0.03783847391605377, "cos_r_negg": [0.5712009072303772, 0.5938788652420044, 0.5852274298667908], "cos_innovation_negg": [0.5712009072303772, 0.5938788652420044, 0.5852274298667908], "cos_apical_negg": [0.0588875412940979, 0.09509463608264923, 0.14293599128723145], "cos_Ac_negg": [0.5924674272537231, 0.6127462983131409, 0.5921125411987305], "r_norm": [0.23549771308898926, 0.23063766956329346, 0.22743842005729675]}, {"step": 2700, "epoch": 5, "train_loss": 0.025878164917230606, "cos_r_negg": [0.6008349061012268, 0.6140125393867493, 0.5990324020385742], "cos_innovation_negg": [0.6008349061012268, 0.6140125393867493, 0.5990324020385742], "cos_apical_negg": [0.04953507333993912, 0.07712547481060028, 0.13934241235256195], "cos_Ac_negg": [0.6149699091911316, 0.6274121999740601, 0.6048215627670288], "r_norm": [0.17445026338100433, 0.17008483409881592, 0.16803543269634247]}, {"step": 2800, "epoch": 5, "train_loss": 0.06025358662009239, "cos_r_negg": [0.5873588919639587, 0.6082230806350708, 0.5916528701782227], "cos_innovation_negg": [0.5873588919639587, 0.6082230806350708, 0.5916528701782227], "cos_apical_negg": [0.03817795217037201, 0.08858905732631683, 0.13129693269729614], "cos_Ac_negg": [0.6018168926239014, 0.6221031546592712, 0.6005264520645142], "r_norm": [0.21433258056640625, 0.20842435956001282, 0.2057434469461441]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9705, "test_loss": 0.11484633636474609}, {"step": 2900, "epoch": 6, "train_loss": 0.005533664021641016, "cos_r_negg": [0.5634152293205261, 0.5864613056182861, 0.5864205360412598], "cos_innovation_negg": [0.5634152293205261, 0.5864613056182861, 0.5864205360412598], "cos_apical_negg": [0.05011267215013504, 0.07456262409687042, 0.13208502531051636], "cos_Ac_negg": [0.5883058905601501, 0.6146793961524963, 0.6048269271850586], "r_norm": [0.21608708798885345, 0.2090061902999878, 0.20696677267551422]}, {"step": 3000, "epoch": 6, "train_loss": 0.04362904280424118, "cos_r_negg": [0.5723827481269836, 0.5900061130523682, 0.5797159075737], "cos_innovation_negg": [0.5723827481269836, 0.5900061130523682, 0.5797159075737], "cos_apical_negg": [0.05936134606599808, 0.06955280900001526, 0.13168014585971832], "cos_Ac_negg": [0.601454496383667, 0.6238418817520142, 0.6006356477737427], "r_norm": [0.18368461728096008, 0.17956972122192383, 0.1773810237646103], "ldr": {"dL_sdil": -0.005626410245895386, "dL_bp": -0.007024504244327545, "ratio": 0.8009690150644931}}, {"step": 3100, "epoch": 6, "train_loss": 0.015955157577991486, "cos_r_negg": [0.5732883214950562, 0.5980974435806274, 0.5891245603561401], "cos_innovation_negg": [0.5732883214950562, 0.5980974435806274, 0.5891245603561401], "cos_apical_negg": [0.041265539824962616, 0.07053113728761673, 0.12908563017845154], "cos_Ac_negg": [0.6080539226531982, 0.6350600719451904, 0.6158679723739624], "r_norm": [0.1764211803674698, 0.17159801721572876, 0.1705530285835266]}, {"step": 3200, "epoch": 6, "train_loss": 0.025097154080867767, "cos_r_negg": [0.5631382465362549, 0.5797849893569946, 0.5840267539024353], "cos_innovation_negg": [0.5631382465362549, 0.5797849893569946, 0.5840267539024353], "cos_apical_negg": [0.02611123025417328, 0.06582914292812347, 0.12043886631727219], "cos_Ac_negg": [0.6107301712036133, 0.6322115063667297, 0.6230251789093018], "r_norm": [0.1865541636943817, 0.1821870654821396, 0.18060094118118286]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9701, "test_loss": 0.12378705406188965}, {"step": 3300, "epoch": 7, "train_loss": 0.029362596571445465, "cos_r_negg": [0.5528621673583984, 0.5784894227981567, 0.577578067779541], "cos_innovation_negg": [0.5528621673583984, 0.5784894227981567, 0.577578067779541], "cos_apical_negg": [0.034285690635442734, 0.06607280671596527, 0.1244848370552063], "cos_Ac_negg": [0.6092706918716431, 0.6358908414840698, 0.6264286637306213], "r_norm": [0.19721956551074982, 0.19170287251472473, 0.19004429876804352]}, {"step": 3400, "epoch": 7, "train_loss": 0.014400985091924667, "cos_r_negg": [0.5379105806350708, 0.5579407215118408, 0.5657634139060974], "cos_innovation_negg": [0.5379105806350708, 0.5579407215118408, 0.5657634139060974], "cos_apical_negg": [0.036192864179611206, 0.0702303797006607, 0.12546120584011078], "cos_Ac_negg": [0.6015320420265198, 0.6361156702041626, 0.6272715330123901], "r_norm": [0.20590651035308838, 0.20184734463691711, 0.19857719540596008]}, {"step": 3500, "epoch": 7, "train_loss": 0.004858485423028469, "cos_r_negg": [0.5217562913894653, 0.5577101707458496, 0.5610089898109436], "cos_innovation_negg": [0.5217562913894653, 0.5577101707458496, 0.5610089898109436], "cos_apical_negg": [0.02481369487941265, 0.06895075738430023, 0.12296687066555023], "cos_Ac_negg": [0.602568507194519, 0.6409847736358643, 0.6339326500892639], "r_norm": [0.16456836462020874, 0.16079244017601013, 0.16020901501178741], "ldr": {"dL_sdil": -0.005762666463851929, "dL_bp": -0.008025065064430237, "ratio": 0.7180834569670952}}, {"step": 3600, "epoch": 7, "train_loss": 0.002303030574694276, "cos_r_negg": [0.5303589105606079, 0.5598253607749939, 0.5574865937232971], "cos_innovation_negg": [0.5303589105606079, 0.5598253607749939, 0.5574865937232971], "cos_apical_negg": [0.028562750667333603, 0.07188724726438522, 0.11762115359306335], "cos_Ac_negg": [0.6107902526855469, 0.646130383014679, 0.6324824094772339], "r_norm": [0.1928023099899292, 0.18809273838996887, 0.18613044917583466]}, {"step": 3700, "epoch": 7, "train_loss": 0.012182761915028095, "cos_r_negg": [0.5196666121482849, 0.5439647436141968, 0.5465419292449951], "cos_innovation_negg": [0.5196666121482849, 0.5439647436141968, 0.5465419292449951], "cos_apical_negg": [0.02880164422094822, 0.061837583780288696, 0.12163949757814407], "cos_Ac_negg": [0.6145820617675781, 0.6401582360267639, 0.629977822303772], "r_norm": [0.18259994685649872, 0.17761272192001343, 0.17525437474250793]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9723, "test_loss": 0.12099323158264161}, {"step": 3800, "epoch": 8, "train_loss": 0.006128543522208929, "cos_r_negg": [0.5074456930160522, 0.541096568107605, 0.5479250550270081], "cos_innovation_negg": [0.5074456930160522, 0.541096568107605, 0.5479250550270081], "cos_apical_negg": [0.03880160301923752, 0.06963154673576355, 0.12138818204402924], "cos_Ac_negg": [0.6084407567977905, 0.6394813060760498, 0.6389426589012146], "r_norm": [0.18895649909973145, 0.18485723435878754, 0.18205872178077698]}, {"step": 3900, "epoch": 8, "train_loss": 0.046840276569128036, "cos_r_negg": [0.4817640781402588, 0.512580394744873, 0.5277144312858582], "cos_innovation_negg": [0.4817640781402588, 0.512580394744873, 0.5277144312858582], "cos_apical_negg": [0.03252935782074928, 0.06808394938707352, 0.11480497568845749], "cos_Ac_negg": [0.5976804494857788, 0.6336661577224731, 0.6421864628791809], "r_norm": [0.1611471027135849, 0.15656572580337524, 0.1543990969657898]}, {"step": 4000, "epoch": 8, "train_loss": 0.010303596965968609, "cos_r_negg": [0.46032238006591797, 0.490315705537796, 0.5095285773277283], "cos_innovation_negg": [0.46032238006591797, 0.490315705537796, 0.5095285773277283], "cos_apical_negg": [0.02937757596373558, 0.06782662123441696, 0.1145288273692131], "cos_Ac_negg": [0.5953971147537231, 0.6307305693626404, 0.6428290605545044], "r_norm": [0.16530068218708038, 0.16076159477233887, 0.15909411013126373], "ldr": {"dL_sdil": -0.006056338548660278, "dL_bp": -0.009109683334827423, "ratio": 0.6648242673272914}}, {"step": 4100, "epoch": 8, "train_loss": 0.006154875736683607, "cos_r_negg": [0.4616425633430481, 0.4977800250053406, 0.517359733581543], "cos_innovation_negg": [0.4616425633430481, 0.4977800250053406, 0.517359733581543], "cos_apical_negg": [0.033913906663656235, 0.07474394142627716, 0.11754339933395386], "cos_Ac_negg": [0.605496883392334, 0.6388245820999146, 0.6508820056915283], "r_norm": [0.20106157660484314, 0.19499371945858002, 0.19201216101646423]}, {"step": 4200, "epoch": 8, "train_loss": 0.027423150837421417, "cos_r_negg": [0.4457210600376129, 0.475233256816864, 0.4958668053150177], "cos_innovation_negg": [0.4457210600376129, 0.475233256816864, 0.4958668053150177], "cos_apical_negg": [0.02756674960255623, 0.06067146733403206, 0.11335664987564087], "cos_Ac_negg": [0.6091917157173157, 0.6434727907180786, 0.6467041969299316], "r_norm": [0.17253616452217102, 0.16870473325252533, 0.16614750027656555]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.972, "test_loss": 0.13293666820526123}, {"step": 4300, "epoch": 9, "train_loss": 0.008598307147622108, "cos_r_negg": [0.43383729457855225, 0.4656917452812195, 0.48779743909835815], "cos_innovation_negg": [0.43383729457855225, 0.4656917452812195, 0.48779743909835815], "cos_apical_negg": [0.025223735719919205, 0.060846418142318726, 0.10914748162031174], "cos_Ac_negg": [0.6169029474258423, 0.646248459815979, 0.6523618102073669], "r_norm": [0.18382036685943604, 0.18095865845680237, 0.17769470810890198]}, {"step": 4400, "epoch": 9, "train_loss": 0.00196667923592031, "cos_r_negg": [0.41872847080230713, 0.452251136302948, 0.47875547409057617], "cos_innovation_negg": [0.41872847080230713, 0.452251136302948, 0.47875547409057617], "cos_apical_negg": [0.02673802338540554, 0.059494443237781525, 0.105801060795784], "cos_Ac_negg": [0.605011522769928, 0.6423466205596924, 0.6571727395057678], "r_norm": [0.17376244068145752, 0.17010605335235596, 0.16702912747859955]}, {"step": 4500, "epoch": 9, "train_loss": 0.025235939770936966, "cos_r_negg": [0.402266263961792, 0.44017136096954346, 0.4623959958553314], "cos_innovation_negg": [0.402266263961792, 0.44017136096954346, 0.4623959958553314], "cos_apical_negg": [0.024557732045650482, 0.05936041101813316, 0.10745266079902649], "cos_Ac_negg": [0.6089265942573547, 0.641944944858551, 0.6555761098861694], "r_norm": [0.16229242086410522, 0.1578385978937149, 0.15560832619667053], "ldr": {"dL_sdil": -0.008048959076404572, "dL_bp": -0.013988755643367767, "ratio": 0.5753877815587319}}, {"step": 4600, "epoch": 9, "train_loss": 0.0004674484662245959, "cos_r_negg": [0.41375449299812317, 0.44948911666870117, 0.46787434816360474], "cos_innovation_negg": [0.41375449299812317, 0.44948911666870117, 0.46787434816360474], "cos_apical_negg": [0.0270821712911129, 0.06542938202619553, 0.11439449340105057], "cos_Ac_negg": [0.6017415523529053, 0.6417720317840576, 0.6538240909576416], "r_norm": [0.18058077991008759, 0.17527592182159424, 0.17288991808891296]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.972, "test_loss": 0.1355734146118164}, {"step": 4700, "epoch": 10, "train_loss": 0.0031531297136098146, "cos_r_negg": [0.38859570026397705, 0.4213126301765442, 0.4452584981918335], "cos_innovation_negg": [0.38859570026397705, 0.4213126301765442, 0.4452584981918335], "cos_apical_negg": [0.022557012736797333, 0.06057509407401085, 0.10062366724014282], "cos_Ac_negg": [0.6080507040023804, 0.6411598324775696, 0.6576842069625854], "r_norm": [0.18241015076637268, 0.1783653050661087, 0.17502665519714355]}, {"step": 4800, "epoch": 10, "train_loss": 0.001424507237970829, "cos_r_negg": [0.37501198053359985, 0.409299373626709, 0.4334845542907715], "cos_innovation_negg": [0.37501198053359985, 0.409299373626709, 0.4334845542907715], "cos_apical_negg": [0.025177378207445145, 0.061661191284656525, 0.10165520012378693], "cos_Ac_negg": [0.5981765985488892, 0.6386995911598206, 0.6585326790809631], "r_norm": [0.18406125903129578, 0.17913511395454407, 0.17667512595653534]}, {"step": 4900, "epoch": 10, "train_loss": 0.002842707559466362, "cos_r_negg": [0.38084954023361206, 0.4200208783149719, 0.44367271661758423], "cos_innovation_negg": [0.38084954023361206, 0.4200208783149719, 0.44367271661758423], "cos_apical_negg": [0.025119103491306305, 0.061888497322797775, 0.10602675378322601], "cos_Ac_negg": [0.5990905165672302, 0.6428531408309937, 0.6620258688926697], "r_norm": [0.176680326461792, 0.17173907160758972, 0.16918231546878815]}, {"step": 5000, "epoch": 10, "train_loss": 0.0005123436567373574, "cos_r_negg": [0.3533342778682709, 0.3915327787399292, 0.41553783416748047], "cos_innovation_negg": [0.3533342778682709, 0.3915327787399292, 0.41553783416748047], "cos_apical_negg": [0.01720595546066761, 0.05739036202430725, 0.09578986465930939], "cos_Ac_negg": [0.5992639064788818, 0.6391037702560425, 0.6600940227508545], "r_norm": [0.19456928968429565, 0.19128933548927307, 0.1872979998588562], "ldr": {"dL_sdil": -0.010446697473526001, "dL_bp": -0.018925122916698456, "ratio": 0.5520015653007161}}, {"step": 5100, "epoch": 10, "train_loss": 0.0036637960001826286, "cos_r_negg": [0.3606067895889282, 0.39345288276672363, 0.4226773977279663], "cos_innovation_negg": [0.3606067895889282, 0.39345288276672363, 0.4226773977279663], "cos_apical_negg": [0.022676775231957436, 0.05807805061340332, 0.10057053714990616], "cos_Ac_negg": [0.6026651859283447, 0.6419999003410339, 0.6661624908447266], "r_norm": [0.17673218250274658, 0.17317934334278107, 0.16993945837020874]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9734, "test_loss": 0.1404675461769104}, {"step": 5200, "epoch": 11, "train_loss": 0.0005994847742840648, "cos_r_negg": [0.35273417830467224, 0.3868192434310913, 0.41649049520492554], "cos_innovation_negg": [0.35273417830467224, 0.3868192434310913, 0.41649049520492554], "cos_apical_negg": [0.01789838634431362, 0.05813433229923248, 0.0956256240606308], "cos_Ac_negg": [0.6013449430465698, 0.6412538290023804, 0.6646535992622375], "r_norm": [0.17028164863586426, 0.16714194416999817, 0.1640455722808838]}, {"step": 5300, "epoch": 11, "train_loss": 0.006956327706575394, "cos_r_negg": [0.3471260368824005, 0.38593626022338867, 0.41008174419403076], "cos_innovation_negg": [0.3471260368824005, 0.38593626022338867, 0.41008174419403076], "cos_apical_negg": [0.022661101073026657, 0.06154236942529678, 0.09650105237960815], "cos_Ac_negg": [0.5949249863624573, 0.6413847208023071, 0.6650874614715576], "r_norm": [0.18341776728630066, 0.17986181378364563, 0.17660996317863464]}, {"step": 5400, "epoch": 11, "train_loss": 0.001547288615256548, "cos_r_negg": [0.33000773191452026, 0.3681381344795227, 0.39361703395843506], "cos_innovation_negg": [0.33000773191452026, 0.3681381344795227, 0.39361703395843506], "cos_apical_negg": [0.0172126442193985, 0.05573862046003342, 0.09154460579156876], "cos_Ac_negg": [0.5914196968078613, 0.6341928839683533, 0.6608392000198364], "r_norm": [0.1795005202293396, 0.17573903501033783, 0.17266663908958435]}, {"step": 5500, "epoch": 11, "train_loss": 0.0010484072845429182, "cos_r_negg": [0.32509326934814453, 0.3683776259422302, 0.39212900400161743], "cos_innovation_negg": [0.32509326934814453, 0.3683776259422302, 0.39212900400161743], "cos_apical_negg": [0.0187190193682909, 0.06007537618279457, 0.09168079495429993], "cos_Ac_negg": [0.5898714661598206, 0.6387882232666016, 0.665573000907898], "r_norm": [0.18720915913581848, 0.18333867192268372, 0.18037936091423035], "ldr": {"dL_sdil": -0.010868288576602936, "dL_bp": -0.020501919090747833, "ratio": 0.530110792482232}}, {"step": 5600, "epoch": 11, "train_loss": 0.003246604697778821, "cos_r_negg": [0.3071094751358032, 0.3475777208805084, 0.3725828230381012], "cos_innovation_negg": [0.3071094751358032, 0.3475777208805084, 0.3725828230381012], "cos_apical_negg": [0.015700895339250565, 0.0529693067073822, 0.0874062031507492], "cos_Ac_negg": [0.5879708528518677, 0.6322216987609863, 0.6637140512466431], "r_norm": [0.17360228300094604, 0.1698247492313385, 0.16715270280838013]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9733, "test_loss": 0.14312982892990112}, {"step": 5700, "epoch": 12, "train_loss": 0.0006515533896163106, "cos_r_negg": [0.3087655305862427, 0.34727153182029724, 0.37167757749557495], "cos_innovation_negg": [0.3087655305862427, 0.34727153182029724, 0.37167757749557495], "cos_apical_negg": [0.0184798464179039, 0.05521545186638832, 0.08523423224687576], "cos_Ac_negg": [0.5891385078430176, 0.6350855827331543, 0.6646750569343567], "r_norm": [0.1833624541759491, 0.1796121597290039, 0.1768713742494583]}, {"step": 5800, "epoch": 12, "train_loss": 0.0003087636723648757, "cos_r_negg": [0.30396589636802673, 0.34521055221557617, 0.37020623683929443], "cos_innovation_negg": [0.30396589636802673, 0.34521055221557617, 0.37020623683929443], "cos_apical_negg": [0.016882814466953278, 0.05496159568428993, 0.08495524525642395], "cos_Ac_negg": [0.5892714858055115, 0.6374879479408264, 0.6660394072532654], "r_norm": [0.17786654829978943, 0.17419278621673584, 0.17145559191703796]}, {"step": 5900, "epoch": 12, "train_loss": 0.0011283843778073788, "cos_r_negg": [0.306083083152771, 0.3477693796157837, 0.3714534342288971], "cos_innovation_negg": [0.306083083152771, 0.3477693796157837, 0.3714534342288971], "cos_apical_negg": [0.018021147698163986, 0.05589523911476135, 0.08503801375627518], "cos_Ac_negg": [0.5883532166481018, 0.6385139226913452, 0.6678342819213867], "r_norm": [0.17690302431583405, 0.17339177429676056, 0.17059600353240967]}, {"step": 6000, "epoch": 12, "train_loss": 0.00048075226368382573, "cos_r_negg": [0.29833588004112244, 0.33743441104888916, 0.3617570400238037], "cos_innovation_negg": [0.29833588004112244, 0.33743441104888916, 0.3617570400238037], "cos_apical_negg": [0.019266171380877495, 0.05505426600575447, 0.08429230749607086], "cos_Ac_negg": [0.5897235870361328, 0.6371286511421204, 0.6667823791503906], "r_norm": [0.17418819665908813, 0.17051303386688232, 0.1679476797580719], "ldr": {"dL_sdil": -0.010284975171089172, "dL_bp": -0.02005515992641449, "ratio": 0.5128343632674259}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9726, "test_loss": 0.14664524269104004}, {"step": 6100, "epoch": 13, "train_loss": 0.0006032318924553692, "cos_r_negg": [0.2905581295490265, 0.3290700316429138, 0.35489749908447266], "cos_innovation_negg": [0.2905581295490265, 0.3290700316429138, 0.35489749908447266], "cos_apical_negg": [0.01673714816570282, 0.053014423698186874, 0.0824136883020401], "cos_Ac_negg": [0.5891910195350647, 0.6343379020690918, 0.6674453616142273], "r_norm": [0.17484882473945618, 0.17138946056365967, 0.16872015595436096]}, {"step": 6200, "epoch": 13, "train_loss": 0.0008048104937188327, "cos_r_negg": [0.28783735632896423, 0.3279610276222229, 0.35211002826690674], "cos_innovation_negg": [0.28783735632896423, 0.3279610276222229, 0.35211002826690674], "cos_apical_negg": [0.017266422510147095, 0.05395973473787308, 0.0814470648765564], "cos_Ac_negg": [0.5860611200332642, 0.6336036920547485, 0.6678301692008972], "r_norm": [0.17862695455551147, 0.17502078413963318, 0.17230205237865448]}, {"step": 6300, "epoch": 13, "train_loss": 0.001458723098039627, "cos_r_negg": [0.28095054626464844, 0.319520503282547, 0.3462580740451813], "cos_innovation_negg": [0.28095054626464844, 0.319520503282547, 0.3462580740451813], "cos_apical_negg": [0.015333285555243492, 0.05295852944254875, 0.08145970851182938], "cos_Ac_negg": [0.5885621905326843, 0.6347628235816956, 0.667636513710022], "r_norm": [0.17873218655586243, 0.17528963088989258, 0.17260552942752838]}, {"step": 6400, "epoch": 13, "train_loss": 0.0009553181007504463, "cos_r_negg": [0.28301292657852173, 0.3196304440498352, 0.3458683490753174], "cos_innovation_negg": [0.28301292657852173, 0.3196304440498352, 0.3458683490753174], "cos_apical_negg": [0.016829241067171097, 0.05420325696468353, 0.08162521570920944], "cos_Ac_negg": [0.5902488231658936, 0.6372597217559814, 0.66744065284729], "r_norm": [0.18012583255767822, 0.17678941786289215, 0.17403095960617065]}, {"step": 6500, "epoch": 13, "train_loss": 0.0001677384425420314, "cos_r_negg": [0.2745632529258728, 0.3147183656692505, 0.33949393033981323], "cos_innovation_negg": [0.2745632529258728, 0.3147183656692505, 0.33949393033981323], "cos_apical_negg": [0.016230184584856033, 0.053433649241924286, 0.08032813668251038], "cos_Ac_negg": [0.5861450433731079, 0.6342117786407471, 0.6667425632476807], "r_norm": [0.17428138852119446, 0.17074427008628845, 0.1682063192129135], "ldr": {"dL_sdil": -0.010724306106567383, "dL_bp": -0.02085144817829132, "ratio": 0.5143194858634605}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9725, "test_loss": 0.15084042530059816}, {"step": 6600, "epoch": 14, "train_loss": 0.0004299550782889128, "cos_r_negg": [0.273845911026001, 0.31272608041763306, 0.3383336663246155], "cos_innovation_negg": [0.273845911026001, 0.31272608041763306, 0.3383336663246155], "cos_apical_negg": [0.01675397902727127, 0.05444930121302605, 0.07982146739959717], "cos_Ac_negg": [0.5896521806716919, 0.6363834142684937, 0.667066216468811], "r_norm": [0.17700453102588654, 0.1736709177494049, 0.17096003890037537]}, {"step": 6700, "epoch": 14, "train_loss": 0.0005057700327597558, "cos_r_negg": [0.2730553150177002, 0.31345319747924805, 0.3370913565158844], "cos_innovation_negg": [0.2730553150177002, 0.31345319747924805, 0.3370913565158844], "cos_apical_negg": [0.016011910513043404, 0.055612288415431976, 0.08066101372241974], "cos_Ac_negg": [0.5874826312065125, 0.6363601684570312, 0.6681541800498962], "r_norm": [0.1798991858959198, 0.17647922039031982, 0.17376676201820374]}, {"step": 6800, "epoch": 14, "train_loss": 0.0006526593351736665, "cos_r_negg": [0.2665591239929199, 0.30510541796684265, 0.3292950987815857], "cos_innovation_negg": [0.2665591239929199, 0.30510541796684265, 0.3292950987815857], "cos_apical_negg": [0.01495925523340702, 0.05403262749314308, 0.07963677495718002], "cos_Ac_negg": [0.5866935849189758, 0.6354987621307373, 0.6675518751144409], "r_norm": [0.18175745010375977, 0.1781677007675171, 0.17535631358623505]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005965807940810919, "cos_r_negg": [0.26459217071533203, 0.30316829681396484, 0.32873469591140747], "cos_innovation_negg": [0.26459217071533203, 0.30316829681396484, 0.32873469591140747], "cos_apical_negg": [0.01662355661392212, 0.054152961820364, 0.07870881259441376], "cos_Ac_negg": [0.5859885215759277, 0.6350167393684387, 0.6674662828445435], "r_norm": [0.17693497240543365, 0.17355331778526306, 0.17087683081626892]}, {"step": 7000, "epoch": 14, "train_loss": 0.0012752392794936895, "cos_r_negg": [0.26172104477882385, 0.3006412088871002, 0.3232826590538025], "cos_innovation_negg": [0.26172104477882385, 0.3006412088871002, 0.3232826590538025], "cos_apical_negg": [0.017376698553562164, 0.054388709366321564, 0.07631789892911911], "cos_Ac_negg": [0.5867894887924194, 0.6361160278320312, 0.6681196689605713], "r_norm": [0.17798875272274017, 0.17451174557209015, 0.17195779085159302], "ldr": {"dL_sdil": -0.011315062642097473, "dL_bp": -0.02217201143503189, "ratio": 0.5103309041334706}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9727, "test_loss": 0.1537931764602661}], "final": {"test_acc": 0.9727, "test_loss": 0.1537931764602661, "wall_s": 23.313454627990723, "cos_r_negg": [0.26067402958869934, 0.2989043891429901, 0.3237406611442566], "cos_innovation_negg": [0.26067402958869934, 0.2989043891429901, 0.3237406611442566], "cos_apical_negg": [0.01768970862030983, 0.053484197705984116, 0.07788942754268646], "cos_Ac_negg": [0.5858397483825684, 0.6353812217712402, 0.6680448055267334]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_residual_s1.json b/results/nuis_ctrl_v1_rho0p05_residual_s1.json new file mode 100644 index 0000000..036239b --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_residual_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_residual_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.011935683898627758, -0.013785665854811668, -0.005787058733403683], "cos_innovation_negg": [0.011935683898627758, -0.013785665854811668, -0.005787058733403683], "cos_apical_negg": [0.011578163132071495, -0.01364164613187313, -0.006125854793936014], "cos_Ac_negg": [0.0119356419891119, -0.01378566026687622, -0.005787072703242302], "r_norm": [4.739142417907715, 4.79836368560791, 4.788496017456055], "ldr": {"dL_sdil": -0.04240679740905762, "dL_bp": -0.2198801040649414, "ratio": 0.19286327696358013}}, {"step": 100, "epoch": 0, "train_loss": 0.19800172746181488, "cos_r_negg": [0.4595581889152527, 0.36300790309906006, 0.3212670087814331], "cos_innovation_negg": [0.4595581889152527, 0.36300790309906006, 0.3212670087814331], "cos_apical_negg": [0.20975911617279053, 0.16929280757904053, 0.19154170155525208], "cos_Ac_negg": [0.4597140848636627, 0.3632028102874756, 0.32125383615493774], "r_norm": [0.9384980201721191, 0.9418078064918518, 0.9590500593185425]}, {"step": 200, "epoch": 0, "train_loss": 0.3248291611671448, "cos_r_negg": [0.4977818429470062, 0.42747363448143005, 0.3676988482475281], "cos_innovation_negg": [0.4977818429470062, 0.42747363448143005, 0.3676988482475281], "cos_apical_negg": [0.20688921213150024, 0.18004465103149414, 0.19594287872314453], "cos_Ac_negg": [0.4978838562965393, 0.42765089869499207, 0.3677062690258026], "r_norm": [0.7502566576004028, 0.7576274871826172, 0.7664961814880371]}, {"step": 300, "epoch": 0, "train_loss": 0.17866581678390503, "cos_r_negg": [0.5077928304672241, 0.4503347873687744, 0.4130593538284302], "cos_innovation_negg": [0.5077928304672241, 0.4503347873687744, 0.4130593538284302], "cos_apical_negg": [0.166174978017807, 0.16370157897472382, 0.1931830197572708], "cos_Ac_negg": [0.5077493190765381, 0.45025303959846497, 0.4126030206680298], "r_norm": [0.582416296005249, 0.5849134922027588, 0.5949047803878784]}, {"step": 400, "epoch": 0, "train_loss": 0.10736608505249023, "cos_r_negg": [0.5154123306274414, 0.46785473823547363, 0.4282694458961487], "cos_innovation_negg": [0.5154123306274414, 0.46785473823547363, 0.4282694458961487], "cos_apical_negg": [0.1695859581232071, 0.16620805859565735, 0.1936672031879425], "cos_Ac_negg": [0.5154713988304138, 0.46782737970352173, 0.4281441271305084], "r_norm": [0.566364049911499, 0.5695450305938721, 0.5781142711639404]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9461, "test_loss": 0.1800264015197754}, {"step": 500, "epoch": 1, "train_loss": 0.16453297436237335, "cos_r_negg": [0.5244979858398438, 0.4845229387283325, 0.43385422229766846], "cos_innovation_negg": [0.5244979858398438, 0.4845229387283325, 0.43385422229766846], "cos_apical_negg": [0.13939255475997925, 0.14868509769439697, 0.19115450978279114], "cos_Ac_negg": [0.5251206159591675, 0.48533526062965393, 0.4335957467556], "r_norm": [0.48811590671539307, 0.4888215661048889, 0.4921705722808838], "ldr": {"dL_sdil": -0.011702954769134521, "dL_bp": -0.00984230637550354, "ratio": 1.1890459738442951}}, {"step": 600, "epoch": 1, "train_loss": 0.10629040747880936, "cos_r_negg": [0.5180792808532715, 0.48784932494163513, 0.4331909120082855], "cos_innovation_negg": [0.5180792808532715, 0.48784932494163513, 0.4331909120082855], "cos_apical_negg": [0.12916310131549835, 0.1492573320865631, 0.1820889413356781], "cos_Ac_negg": [0.5184558033943176, 0.48788216710090637, 0.43305861949920654], "r_norm": [0.44508522748947144, 0.4487663507461548, 0.45049262046813965]}, {"step": 700, "epoch": 1, "train_loss": 0.16264066100120544, "cos_r_negg": [0.527536928653717, 0.49619653820991516, 0.4591195583343506], "cos_innovation_negg": [0.527536928653717, 0.49619653820991516, 0.4591195583343506], "cos_apical_negg": [0.13695219159126282, 0.15048012137413025, 0.18260575830936432], "cos_Ac_negg": [0.5289961099624634, 0.4966246485710144, 0.4590521454811096], "r_norm": [0.4516822099685669, 0.4543275237083435, 0.4520670175552368]}, {"step": 800, "epoch": 1, "train_loss": 0.19131723046302795, "cos_r_negg": [0.5477697849273682, 0.5233035087585449, 0.46972203254699707], "cos_innovation_negg": [0.5477697849273682, 0.5233035087585449, 0.46972203254699707], "cos_apical_negg": [0.10759079456329346, 0.14349976181983948, 0.1710243821144104], "cos_Ac_negg": [0.5484983921051025, 0.5231080055236816, 0.46940040588378906], "r_norm": [0.35974693298339844, 0.36176374554634094, 0.3669247627258301]}, {"step": 900, "epoch": 1, "train_loss": 0.23413461446762085, "cos_r_negg": [0.5469974279403687, 0.5189642310142517, 0.4694017767906189], "cos_innovation_negg": [0.5469974279403687, 0.5189642310142517, 0.4694017767906189], "cos_apical_negg": [0.10924465954303741, 0.13199782371520996, 0.1644289791584015], "cos_Ac_negg": [0.5469458103179932, 0.518856406211853, 0.46931537985801697], "r_norm": [0.39200443029403687, 0.39184892177581787, 0.3952348828315735]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9594, "test_loss": 0.14740810661315917}, {"step": 1000, "epoch": 2, "train_loss": 0.020659351721405983, "cos_r_negg": [0.5646075010299683, 0.5425055027008057, 0.4808516502380371], "cos_innovation_negg": [0.5646075010299683, 0.5425055027008057, 0.4808516502380371], "cos_apical_negg": [0.1025833785533905, 0.12162604928016663, 0.15607091784477234], "cos_Ac_negg": [0.564576268196106, 0.5423530340194702, 0.4805763363838196], "r_norm": [0.35054507851600647, 0.3523258864879608, 0.35283398628234863], "ldr": {"dL_sdil": -0.010484561324119568, "dL_bp": -0.009114019572734833, "ratio": 1.1503773105211226}}, {"step": 1100, "epoch": 2, "train_loss": 0.10964615643024445, "cos_r_negg": [0.5504506230354309, 0.5405658483505249, 0.4947567582130432], "cos_innovation_negg": [0.5504506230354309, 0.5405658483505249, 0.4947567582130432], "cos_apical_negg": [0.09591667354106903, 0.1244964525103569, 0.16299867630004883], "cos_Ac_negg": [0.5513126254081726, 0.5401743650436401, 0.49388736486434937], "r_norm": [0.3291342258453369, 0.33226123452186584, 0.3298074007034302]}, {"step": 1200, "epoch": 2, "train_loss": 0.15681979060173035, "cos_r_negg": [0.5681092143058777, 0.5566937923431396, 0.5013537406921387], "cos_innovation_negg": [0.5681092143058777, 0.5566937923431396, 0.5013537406921387], "cos_apical_negg": [0.10015411674976349, 0.11984756588935852, 0.15649977326393127], "cos_Ac_negg": [0.5682443380355835, 0.5563015937805176, 0.5008739233016968], "r_norm": [0.31270861625671387, 0.3139200806617737, 0.3170466125011444]}, {"step": 1300, "epoch": 2, "train_loss": 0.13473723828792572, "cos_r_negg": [0.5784183144569397, 0.5624803304672241, 0.5229552984237671], "cos_innovation_negg": [0.5784183144569397, 0.5624803304672241, 0.5229552984237671], "cos_apical_negg": [0.09875530004501343, 0.12516838312149048, 0.16183607280254364], "cos_Ac_negg": [0.5786624550819397, 0.5625358819961548, 0.5228804349899292], "r_norm": [0.328178346157074, 0.3266424536705017, 0.32696443796157837]}, {"step": 1400, "epoch": 2, "train_loss": 0.09945511817932129, "cos_r_negg": [0.5877261757850647, 0.5833468437194824, 0.5220316648483276], "cos_innovation_negg": [0.5877261757850647, 0.5833468437194824, 0.5220316648483276], "cos_apical_negg": [0.08485932648181915, 0.10183429718017578, 0.1582496017217636], "cos_Ac_negg": [0.5899650454521179, 0.5846056342124939, 0.5218909978866577], "r_norm": [0.26476091146469116, 0.26450884342193604, 0.2684342861175537]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9662, "test_loss": 0.11500913486480713}, {"step": 1500, "epoch": 3, "train_loss": 0.044799480587244034, "cos_r_negg": [0.5729881525039673, 0.5794516801834106, 0.5339221954345703], "cos_innovation_negg": [0.5729881525039673, 0.5794516801834106, 0.5339221954345703], "cos_apical_negg": [0.05709003284573555, 0.10583391040563583, 0.1609860360622406], "cos_Ac_negg": [0.5750377178192139, 0.5808219909667969, 0.5336863994598389], "r_norm": [0.27066415548324585, 0.2718654274940491, 0.2756950855255127], "ldr": {"dL_sdil": -0.0055206939578056335, "dL_bp": -0.005840882658958435, "ratio": 0.9451814528987819}}, {"step": 1600, "epoch": 3, "train_loss": 0.09553006291389465, "cos_r_negg": [0.5864070653915405, 0.591616690158844, 0.5493444204330444], "cos_innovation_negg": [0.5864070653915405, 0.591616690158844, 0.5493444204330444], "cos_apical_negg": [0.06386640667915344, 0.09832250326871872, 0.15193292498588562], "cos_Ac_negg": [0.5889490842819214, 0.5921638607978821, 0.5484168529510498], "r_norm": [0.26048943400382996, 0.2610347867012024, 0.2635071277618408]}, {"step": 1700, "epoch": 3, "train_loss": 0.10861323773860931, "cos_r_negg": [0.5862084031105042, 0.587340235710144, 0.5640803575515747], "cos_innovation_negg": [0.5862084031105042, 0.587340235710144, 0.5640803575515747], "cos_apical_negg": [0.08238239586353302, 0.11745046079158783, 0.16670380532741547], "cos_Ac_negg": [0.5915169715881348, 0.590603232383728, 0.5643587708473206], "r_norm": [0.32203879952430725, 0.31957849860191345, 0.32221710681915283]}, {"step": 1800, "epoch": 3, "train_loss": 0.04907859116792679, "cos_r_negg": [0.5856608152389526, 0.5890840888023376, 0.5482282638549805], "cos_innovation_negg": [0.5856608152389526, 0.5890840888023376, 0.5482282638549805], "cos_apical_negg": [0.06749321520328522, 0.09056306630373001, 0.14532238245010376], "cos_Ac_negg": [0.5878607630729675, 0.5907537341117859, 0.5485868453979492], "r_norm": [0.2197471410036087, 0.2204505205154419, 0.221063032746315]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9672, "test_loss": 0.11288363456726075}, {"step": 1900, "epoch": 4, "train_loss": 0.02294781059026718, "cos_r_negg": [0.5973317623138428, 0.6043586730957031, 0.5593540668487549], "cos_innovation_negg": [0.5973317623138428, 0.6043586730957031, 0.5593540668487549], "cos_apical_negg": [0.0686958059668541, 0.1018339991569519, 0.14995601773262024], "cos_Ac_negg": [0.6019316911697388, 0.6075939536094666, 0.5601412653923035], "r_norm": [0.25126156210899353, 0.2498759925365448, 0.25224488973617554]}, {"step": 2000, "epoch": 4, "train_loss": 0.09122863411903381, "cos_r_negg": [0.5711127519607544, 0.5799358487129211, 0.5689223408699036], "cos_innovation_negg": [0.5711127519607544, 0.5799358487129211, 0.5689223408699036], "cos_apical_negg": [0.04912835359573364, 0.09740493446588516, 0.14483867585659027], "cos_Ac_negg": [0.5858960747718811, 0.5902670621871948, 0.5742981433868408], "r_norm": [0.23155272006988525, 0.22967955470085144, 0.23148378729820251], "ldr": {"dL_sdil": -0.005922161042690277, "dL_bp": -0.006407603621482849, "ratio": 0.9242396054017725}}, {"step": 2100, "epoch": 4, "train_loss": 0.0664985328912735, "cos_r_negg": [0.5840332508087158, 0.6021450757980347, 0.5844618082046509], "cos_innovation_negg": [0.5840332508087158, 0.6021450757980347, 0.5844618082046509], "cos_apical_negg": [0.07027436792850494, 0.10700127482414246, 0.14804022014141083], "cos_Ac_negg": [0.5876871943473816, 0.6038587093353271, 0.5843597650527954], "r_norm": [0.22726424038410187, 0.225623220205307, 0.22834229469299316]}, {"step": 2200, "epoch": 4, "train_loss": 0.023848218843340874, "cos_r_negg": [0.596025288105011, 0.6008343696594238, 0.5844902396202087], "cos_innovation_negg": [0.596025288105011, 0.6008343696594238, 0.5844902396202087], "cos_apical_negg": [0.04419754445552826, 0.08808250725269318, 0.13897162675857544], "cos_Ac_negg": [0.6045669317245483, 0.6077235341072083, 0.5862392783164978], "r_norm": [0.22590483725070953, 0.2255634367465973, 0.22662967443466187]}, {"step": 2300, "epoch": 4, "train_loss": 0.14877982437610626, "cos_r_negg": [0.5799214839935303, 0.5900747776031494, 0.5790226459503174], "cos_innovation_negg": [0.5799214839935303, 0.5900747776031494, 0.5790226459503174], "cos_apical_negg": [0.04342891275882721, 0.0754004716873169, 0.1416110098361969], "cos_Ac_negg": [0.5919167399406433, 0.6006958484649658, 0.5831726789474487], "r_norm": [0.22525566816329956, 0.22617968916893005, 0.22663938999176025]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9692, "test_loss": 0.11152303066253662}, {"step": 2400, "epoch": 5, "train_loss": 0.07470186054706573, "cos_r_negg": [0.5891973972320557, 0.6044924259185791, 0.587180495262146], "cos_innovation_negg": [0.5891973972320557, 0.6044924259185791, 0.587180495262146], "cos_apical_negg": [0.040368471294641495, 0.06973414123058319, 0.13383647799491882], "cos_Ac_negg": [0.6007899641990662, 0.6130261421203613, 0.5905767679214478], "r_norm": [0.2216203510761261, 0.22197623550891876, 0.222914919257164]}, {"step": 2500, "epoch": 5, "train_loss": 0.02789377048611641, "cos_r_negg": [0.5571001768112183, 0.5795426368713379, 0.5836772918701172], "cos_innovation_negg": [0.5571001768112183, 0.5795426368713379, 0.5836772918701172], "cos_apical_negg": [0.03303978219628334, 0.07389634102582932, 0.1359260380268097], "cos_Ac_negg": [0.5869741439819336, 0.6067754626274109, 0.5961843132972717], "r_norm": [0.18438225984573364, 0.1837213933467865, 0.18523038923740387], "ldr": {"dL_sdil": -0.005010098218917847, "dL_bp": -0.006476372480392456, "ratio": 0.7735963664977843}}, {"step": 2600, "epoch": 5, "train_loss": 0.06555721163749695, "cos_r_negg": [0.5787664651870728, 0.6029772758483887, 0.5920038223266602], "cos_innovation_negg": [0.5787664651870728, 0.6029772758483887, 0.5920038223266602], "cos_apical_negg": [0.054474882781505585, 0.09032820165157318, 0.13570192456245422], "cos_Ac_negg": [0.6060702800750732, 0.6261075735092163, 0.6057282090187073], "r_norm": [0.2206229269504547, 0.22138342261314392, 0.2254534661769867]}, {"step": 2700, "epoch": 5, "train_loss": 0.04593146592378616, "cos_r_negg": [0.6017680764198303, 0.6105853915214539, 0.5990234613418579], "cos_innovation_negg": [0.6017680764198303, 0.6105853915214539, 0.5990234613418579], "cos_apical_negg": [0.030534956604242325, 0.07209170609712601, 0.12952500581741333], "cos_Ac_negg": [0.6265010237693787, 0.6308919787406921, 0.6108779907226562], "r_norm": [0.17367111146450043, 0.17301763594150543, 0.1753917634487152]}, {"step": 2800, "epoch": 5, "train_loss": 0.11331872642040253, "cos_r_negg": [0.5852015018463135, 0.6050297021865845, 0.6058430671691895], "cos_innovation_negg": [0.5852015018463135, 0.6050297021865845, 0.6058430671691895], "cos_apical_negg": [0.037248946726322174, 0.08929359912872314, 0.1452261209487915], "cos_Ac_negg": [0.620559811592102, 0.6386481523513794, 0.6224511861801147], "r_norm": [0.1954781711101532, 0.19395135343074799, 0.19528909027576447]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9713, "test_loss": 0.10515918369293213}, {"step": 2900, "epoch": 6, "train_loss": 0.006533927284181118, "cos_r_negg": [0.5618231892585754, 0.5823687314987183, 0.5865188837051392], "cos_innovation_negg": [0.5618231892585754, 0.5823687314987183, 0.5865188837051392], "cos_apical_negg": [0.035457197576761246, 0.08563341200351715, 0.14092126488685608], "cos_Ac_negg": [0.6109321117401123, 0.6287125945091248, 0.6156300902366638], "r_norm": [0.1872854232788086, 0.18649417161941528, 0.18763405084609985]}, {"step": 3000, "epoch": 6, "train_loss": 0.08300692588090897, "cos_r_negg": [0.5744534730911255, 0.5794456005096436, 0.5861250162124634], "cos_innovation_negg": [0.5744534730911255, 0.5794456005096436, 0.5861250162124634], "cos_apical_negg": [0.04058567434549332, 0.0793231651186943, 0.13168630003929138], "cos_Ac_negg": [0.6225829720497131, 0.6238137483596802, 0.6147491931915283], "r_norm": [0.1884726881980896, 0.18706244230270386, 0.18900558352470398], "ldr": {"dL_sdil": -0.005829624831676483, "dL_bp": -0.00748685747385025, "ratio": 0.7786477640369043}}, {"step": 3100, "epoch": 6, "train_loss": 0.013627316802740097, "cos_r_negg": [0.5783639550209045, 0.584351658821106, 0.5909626483917236], "cos_innovation_negg": [0.5783639550209045, 0.584351658821106, 0.5909626483917236], "cos_apical_negg": [0.02789091132581234, 0.07112062722444534, 0.13366471230983734], "cos_Ac_negg": [0.6294291019439697, 0.6274665594100952, 0.6191610097885132], "r_norm": [0.14944246411323547, 0.15006566047668457, 0.15108169615268707]}, {"step": 3200, "epoch": 6, "train_loss": 0.04405490309000015, "cos_r_negg": [0.5540293455123901, 0.5780806541442871, 0.5798403024673462], "cos_innovation_negg": [0.5540293455123901, 0.5780806541442871, 0.5798403024673462], "cos_apical_negg": [0.024207837879657745, 0.06950204074382782, 0.1315123736858368], "cos_Ac_negg": [0.6184898018836975, 0.6359472274780273, 0.6188316345214844], "r_norm": [0.1712276190519333, 0.1713661104440689, 0.17324110865592957]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9721, "test_loss": 0.11621832408905029}, {"step": 3300, "epoch": 7, "train_loss": 0.013389818370342255, "cos_r_negg": [0.5641270279884338, 0.5790151953697205, 0.5924917459487915], "cos_innovation_negg": [0.5641270279884338, 0.5790151953697205, 0.5924917459487915], "cos_apical_negg": [0.034241147339344025, 0.07209589332342148, 0.1287578046321869], "cos_Ac_negg": [0.6335006952285767, 0.6423917412757874, 0.6323614716529846], "r_norm": [0.18159674108028412, 0.18028287589550018, 0.18132713437080383]}, {"step": 3400, "epoch": 7, "train_loss": 0.029773881658911705, "cos_r_negg": [0.5414331555366516, 0.555443286895752, 0.5770403742790222], "cos_innovation_negg": [0.5414331555366516, 0.555443286895752, 0.5770403742790222], "cos_apical_negg": [0.031215228140354156, 0.0754883885383606, 0.13154932856559753], "cos_Ac_negg": [0.6293191909790039, 0.6381579041481018, 0.6348534822463989], "r_norm": [0.19872957468032837, 0.1972137689590454, 0.19681642949581146]}, {"step": 3500, "epoch": 7, "train_loss": 0.0030391858890652657, "cos_r_negg": [0.5294433236122131, 0.5440030097961426, 0.572765588760376], "cos_innovation_negg": [0.5294433236122131, 0.5440030097961426, 0.572765588760376], "cos_apical_negg": [0.023815037682652473, 0.06500785052776337, 0.13215318322181702], "cos_Ac_negg": [0.6329994201660156, 0.6423251628875732, 0.6384730339050293], "r_norm": [0.15773600339889526, 0.15748322010040283, 0.15743637084960938], "ldr": {"dL_sdil": -0.005737520754337311, "dL_bp": -0.007938481867313385, "ratio": 0.7227478566099007}}, {"step": 3600, "epoch": 7, "train_loss": 0.00181392056401819, "cos_r_negg": [0.5173041224479675, 0.5402019023895264, 0.5612258911132812], "cos_innovation_negg": [0.5173041224479675, 0.5402019023895264, 0.5612258911132812], "cos_apical_negg": [0.034563735127449036, 0.06581186503171921, 0.13177812099456787], "cos_Ac_negg": [0.623420000076294, 0.6413934230804443, 0.6396375894546509], "r_norm": [0.2031160295009613, 0.20182238519191742, 0.2009257823228836]}, {"step": 3700, "epoch": 7, "train_loss": 0.005001378245651722, "cos_r_negg": [0.5316832661628723, 0.5508448481559753, 0.5710439682006836], "cos_innovation_negg": [0.5316832661628723, 0.5508448481559753, 0.5710439682006836], "cos_apical_negg": [0.032789912074804306, 0.06641100347042084, 0.12860509753227234], "cos_Ac_negg": [0.6304998993873596, 0.6423559188842773, 0.6453312039375305], "r_norm": [0.16545230150222778, 0.16468030214309692, 0.16423511505126953]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9754, "test_loss": 0.11100868663787841}, {"step": 3800, "epoch": 8, "train_loss": 0.0019634724594652653, "cos_r_negg": [0.49285173416137695, 0.522550106048584, 0.5501976013183594], "cos_innovation_negg": [0.49285173416137695, 0.522550106048584, 0.5501976013183594], "cos_apical_negg": [0.018828824162483215, 0.06898509711027145, 0.12255098670721054], "cos_Ac_negg": [0.6295504570007324, 0.643524169921875, 0.6488214731216431], "r_norm": [0.16043861210346222, 0.15994001924991608, 0.16116008162498474]}, {"step": 3900, "epoch": 8, "train_loss": 0.009662585332989693, "cos_r_negg": [0.4753582775592804, 0.5029117465019226, 0.5357325673103333], "cos_innovation_negg": [0.4753582775592804, 0.5029117465019226, 0.5357325673103333], "cos_apical_negg": [0.017303794622421265, 0.061728790402412415, 0.12165973335504532], "cos_Ac_negg": [0.6283164024353027, 0.6399952173233032, 0.6502859592437744], "r_norm": [0.16895166039466858, 0.167774498462677, 0.16917681694030762]}, {"step": 4000, "epoch": 8, "train_loss": 0.012053707614541054, "cos_r_negg": [0.4598022401332855, 0.48842787742614746, 0.5198114514350891], "cos_innovation_negg": [0.4598022401332855, 0.48842787742614746, 0.5198114514350891], "cos_apical_negg": [0.022250913083553314, 0.06144747510552406, 0.12196500599384308], "cos_Ac_negg": [0.6305398941040039, 0.6407707333564758, 0.6521885991096497], "r_norm": [0.16535891592502594, 0.16453412175178528, 0.16724371910095215], "ldr": {"dL_sdil": -0.007221609354019165, "dL_bp": -0.011598825454711914, "ratio": 0.6226155727764189}}, {"step": 4100, "epoch": 8, "train_loss": 0.007018683943897486, "cos_r_negg": [0.46128901839256287, 0.4965739846229553, 0.5250366926193237], "cos_innovation_negg": [0.46128901839256287, 0.4965739846229553, 0.5250366926193237], "cos_apical_negg": [0.018589844927191734, 0.0649154856801033, 0.11730692535638809], "cos_Ac_negg": [0.632196307182312, 0.6469199657440186, 0.6585046648979187], "r_norm": [0.1577710211277008, 0.15650255978107452, 0.15849930047988892]}, {"step": 4200, "epoch": 8, "train_loss": 0.02269434742629528, "cos_r_negg": [0.47296077013015747, 0.5012197494506836, 0.5313794016838074], "cos_innovation_negg": [0.47296077013015747, 0.5012197494506836, 0.5313794016838074], "cos_apical_negg": [0.010709510184824467, 0.056064411997795105, 0.11694499850273132], "cos_Ac_negg": [0.6360347270965576, 0.6495529413223267, 0.6575896143913269], "r_norm": [0.1453334391117096, 0.1442568004131317, 0.14695820212364197]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.973, "test_loss": 0.12612433528900147}, {"step": 4300, "epoch": 9, "train_loss": 0.0055565061047673225, "cos_r_negg": [0.4508157968521118, 0.4840299189090729, 0.5116130709648132], "cos_innovation_negg": [0.4508157968521118, 0.4840299189090729, 0.5116130709648132], "cos_apical_negg": [0.018956102430820465, 0.06528937071561813, 0.11622902750968933], "cos_Ac_negg": [0.6355956196784973, 0.6468340158462524, 0.6545963287353516], "r_norm": [0.15752550959587097, 0.15622948110103607, 0.158013254404068]}, {"step": 4400, "epoch": 9, "train_loss": 0.0009345239959657192, "cos_r_negg": [0.43100717663764954, 0.4660802185535431, 0.4992997944355011], "cos_innovation_negg": [0.43100717663764954, 0.4660802185535431, 0.4992997944355011], "cos_apical_negg": [0.0012484766775742173, 0.05552934855222702, 0.11143714934587479], "cos_Ac_negg": [0.6282694339752197, 0.6399216651916504, 0.6582869291305542], "r_norm": [0.1593485325574875, 0.15823084115982056, 0.16016815602779388]}, {"step": 4500, "epoch": 9, "train_loss": 0.03349297121167183, "cos_r_negg": [0.4123862683773041, 0.45048922300338745, 0.48813825845718384], "cos_innovation_negg": [0.4123862683773041, 0.45048922300338745, 0.48813825845718384], "cos_apical_negg": [0.0073779248632490635, 0.055708736181259155, 0.11663167178630829], "cos_Ac_negg": [0.6335850358009338, 0.6434621810913086, 0.66236412525177], "r_norm": [0.15378838777542114, 0.15259474515914917, 0.15409553050994873], "ldr": {"dL_sdil": -0.007371291518211365, "dL_bp": -0.013118952512741089, "ratio": 0.5618811037735206}}, {"step": 4600, "epoch": 9, "train_loss": 0.001479916274547577, "cos_r_negg": [0.4173874855041504, 0.4497365951538086, 0.4874991774559021], "cos_innovation_negg": [0.4173874855041504, 0.4497365951538086, 0.4874991774559021], "cos_apical_negg": [0.012430883944034576, 0.056131042540073395, 0.11374486982822418], "cos_Ac_negg": [0.6330933570861816, 0.6427613496780396, 0.6594250202178955], "r_norm": [0.1550893485546112, 0.1537916362285614, 0.1546764075756073]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9751, "test_loss": 0.12642392368316652}, {"step": 4700, "epoch": 10, "train_loss": 0.00215778942219913, "cos_r_negg": [0.39477741718292236, 0.43047553300857544, 0.47205987572669983], "cos_innovation_negg": [0.39477741718292236, 0.43047553300857544, 0.47205987572669983], "cos_apical_negg": [0.01019405759871006, 0.05123322829604149, 0.10903597623109818], "cos_Ac_negg": [0.6249011158943176, 0.6408618688583374, 0.6627616882324219], "r_norm": [0.15229037404060364, 0.15106777846813202, 0.15219895541667938]}, {"step": 4800, "epoch": 10, "train_loss": 0.002831760561093688, "cos_r_negg": [0.3821009397506714, 0.42313966155052185, 0.46219125390052795], "cos_innovation_negg": [0.3821009397506714, 0.42313966155052185, 0.46219125390052795], "cos_apical_negg": [0.003800086211413145, 0.052510738372802734, 0.10751186311244965], "cos_Ac_negg": [0.629231333732605, 0.6427340507507324, 0.6645975112915039], "r_norm": [0.14961805939674377, 0.14873483777046204, 0.14977099001407623]}, {"step": 4900, "epoch": 10, "train_loss": 0.001507880399003625, "cos_r_negg": [0.3845168352127075, 0.4220179319381714, 0.46184539794921875], "cos_innovation_negg": [0.3845168352127075, 0.4220179319381714, 0.46184539794921875], "cos_apical_negg": [0.006418270990252495, 0.04843633249402046, 0.10634401440620422], "cos_Ac_negg": [0.6274188756942749, 0.6420944929122925, 0.6620548963546753], "r_norm": [0.1594342142343521, 0.15761792659759521, 0.15847983956336975]}, {"step": 5000, "epoch": 10, "train_loss": 0.0006326170405372977, "cos_r_negg": [0.365106999874115, 0.40706682205200195, 0.4476078450679779], "cos_innovation_negg": [0.365106999874115, 0.40706682205200195, 0.4476078450679779], "cos_apical_negg": [0.0019759703427553177, 0.05384797602891922, 0.10979677736759186], "cos_Ac_negg": [0.6259748339653015, 0.6441224813461304, 0.6630863547325134], "r_norm": [0.15052375197410583, 0.149494469165802, 0.15060998499393463], "ldr": {"dL_sdil": -0.007718577980995178, "dL_bp": -0.014518767595291138, "ratio": 0.5316276281947332}}, {"step": 5100, "epoch": 10, "train_loss": 0.0012461576843634248, "cos_r_negg": [0.35911571979522705, 0.3983779549598694, 0.4413723349571228], "cos_innovation_negg": [0.35911571979522705, 0.3983779549598694, 0.4413723349571228], "cos_apical_negg": [0.002363624982535839, 0.04849231243133545, 0.10589759051799774], "cos_Ac_negg": [0.6248847842216492, 0.6437404751777649, 0.6656257510185242], "r_norm": [0.15122172236442566, 0.1501307189464569, 0.15147460997104645]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9759, "test_loss": 0.1276949785232544}, {"step": 5200, "epoch": 11, "train_loss": 0.0012402110733091831, "cos_r_negg": [0.35025155544281006, 0.3906233012676239, 0.4298197329044342], "cos_innovation_negg": [0.35025155544281006, 0.3906233012676239, 0.4298197329044342], "cos_apical_negg": [0.00504857674241066, 0.052778299897909164, 0.10604824125766754], "cos_Ac_negg": [0.62543785572052, 0.6403063535690308, 0.6640012264251709], "r_norm": [0.14704205095767975, 0.1458543986082077, 0.14721126854419708]}, {"step": 5300, "epoch": 11, "train_loss": 0.0012527708895504475, "cos_r_negg": [0.3447659909725189, 0.3828068673610687, 0.42176806926727295], "cos_innovation_negg": [0.3447659909725189, 0.3828068673610687, 0.42176806926727295], "cos_apical_negg": [0.0029204804450273514, 0.04953660070896149, 0.10046202689409256], "cos_Ac_negg": [0.6246902942657471, 0.6398322582244873, 0.663558840751648], "r_norm": [0.1457543671131134, 0.1447260081768036, 0.1462690830230713]}, {"step": 5400, "epoch": 11, "train_loss": 0.0026527205482125282, "cos_r_negg": [0.3370477557182312, 0.3748803436756134, 0.41634199023246765], "cos_innovation_negg": [0.3370477557182312, 0.3748803436756134, 0.41634199023246765], "cos_apical_negg": [0.0023171482607722282, 0.04821858927607536, 0.09994305670261383], "cos_Ac_negg": [0.622248649597168, 0.6417462825775146, 0.6651083827018738], "r_norm": [0.14654432237148285, 0.14567247033119202, 0.14684513211250305]}, {"step": 5500, "epoch": 11, "train_loss": 0.00147485570050776, "cos_r_negg": [0.3404698073863983, 0.3794620633125305, 0.4205111861228943], "cos_innovation_negg": [0.3404698073863983, 0.3794620633125305, 0.4205111861228943], "cos_apical_negg": [0.004639003425836563, 0.048866137862205505, 0.09987516701221466], "cos_Ac_negg": [0.6238670349121094, 0.6419881582260132, 0.6679314970970154], "r_norm": [0.15389621257781982, 0.15288250148296356, 0.1536381095647812], "ldr": {"dL_sdil": -0.008258819580078125, "dL_bp": -0.01560879498720169, "ratio": 0.5291132074481009}}, {"step": 5600, "epoch": 11, "train_loss": 0.0010807700455188751, "cos_r_negg": [0.32366716861724854, 0.3646085858345032, 0.40230053663253784], "cos_innovation_negg": [0.32366716861724854, 0.3646085858345032, 0.40230053663253784], "cos_apical_negg": [0.003534013871103525, 0.04884061589837074, 0.09961765259504318], "cos_Ac_negg": [0.615607500076294, 0.6380172967910767, 0.6647375226020813], "r_norm": [0.1503227800130844, 0.14935144782066345, 0.150526762008667]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9754, "test_loss": 0.13247075786590576}, {"step": 5700, "epoch": 12, "train_loss": 0.0005582757294178009, "cos_r_negg": [0.32820063829421997, 0.36593544483184814, 0.4054841995239258], "cos_innovation_negg": [0.32820063829421997, 0.36593544483184814, 0.4054841995239258], "cos_apical_negg": [0.0037302712444216013, 0.045040614902973175, 0.09691508114337921], "cos_Ac_negg": [0.6218763589859009, 0.6408955454826355, 0.6676142811775208], "r_norm": [0.1485031694173813, 0.147519052028656, 0.14880996942520142]}, {"step": 5800, "epoch": 12, "train_loss": 0.00036639836616814137, "cos_r_negg": [0.31826362013816833, 0.3544495701789856, 0.3930700719356537], "cos_innovation_negg": [0.31826362013816833, 0.3544495701789856, 0.3930700719356537], "cos_apical_negg": [0.003201207611709833, 0.04491714760661125, 0.09604141116142273], "cos_Ac_negg": [0.616691529750824, 0.6374433040618896, 0.6658985018730164], "r_norm": [0.14986775815486908, 0.14872559905052185, 0.14994212985038757]}, {"step": 5900, "epoch": 12, "train_loss": 0.0026673325337469578, "cos_r_negg": [0.3165438175201416, 0.3531757891178131, 0.3893282413482666], "cos_innovation_negg": [0.3165438175201416, 0.3531757891178131, 0.3893282413482666], "cos_apical_negg": [0.003569494467228651, 0.045752979815006256, 0.0928518995642662], "cos_Ac_negg": [0.6143794059753418, 0.6380530595779419, 0.6660316586494446], "r_norm": [0.14996939897537231, 0.1488775908946991, 0.15008890628814697]}, {"step": 6000, "epoch": 12, "train_loss": 0.00014879496302455664, "cos_r_negg": [0.30961406230926514, 0.3466123640537262, 0.3855319023132324], "cos_innovation_negg": [0.30961406230926514, 0.3466123640537262, 0.3855319023132324], "cos_apical_negg": [0.0015506725758314133, 0.044106483459472656, 0.09334377199411392], "cos_Ac_negg": [0.6153971552848816, 0.6390558481216431, 0.6667139530181885], "r_norm": [0.1464368999004364, 0.14536206424236298, 0.14683833718299866], "ldr": {"dL_sdil": -0.008332058787345886, "dL_bp": -0.016469426453113556, "ratio": 0.5059106831112935}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9753, "test_loss": 0.13651623115539552}, {"step": 6100, "epoch": 13, "train_loss": 0.00037324384902603924, "cos_r_negg": [0.30239158868789673, 0.3397728204727173, 0.37674129009246826], "cos_innovation_negg": [0.30239158868789673, 0.3397728204727173, 0.37674129009246826], "cos_apical_negg": [0.004823541268706322, 0.04640675708651543, 0.09096071124076843], "cos_Ac_negg": [0.6124904751777649, 0.6365949511528015, 0.6662158966064453], "r_norm": [0.14694678783416748, 0.14580979943275452, 0.1474391520023346]}, {"step": 6200, "epoch": 13, "train_loss": 0.0011788607807829976, "cos_r_negg": [0.2936674952507019, 0.3329145908355713, 0.37022751569747925], "cos_innovation_negg": [0.2936674952507019, 0.3329145908355713, 0.37022751569747925], "cos_apical_negg": [0.003545030951499939, 0.04557621479034424, 0.09047915041446686], "cos_Ac_negg": [0.6121279001235962, 0.6354484558105469, 0.6650893092155457], "r_norm": [0.14955121278762817, 0.14838676154613495, 0.15036849677562714]}, {"step": 6300, "epoch": 13, "train_loss": 0.0005334530142135918, "cos_r_negg": [0.2936256527900696, 0.332280695438385, 0.36860334873199463], "cos_innovation_negg": [0.2936256527900696, 0.332280695438385, 0.36860334873199463], "cos_apical_negg": [0.002949201036244631, 0.04521677643060684, 0.08821894973516464], "cos_Ac_negg": [0.6121575236320496, 0.635464072227478, 0.666549563407898], "r_norm": [0.14792223274707794, 0.14679907262325287, 0.14844593405723572]}, {"step": 6400, "epoch": 13, "train_loss": 0.0006172251305542886, "cos_r_negg": [0.28886568546295166, 0.32823634147644043, 0.3629181385040283], "cos_innovation_negg": [0.28886568546295166, 0.32823634147644043, 0.3629181385040283], "cos_apical_negg": [0.001749833463691175, 0.043821848928928375, 0.0855003371834755], "cos_Ac_negg": [0.6091131567955017, 0.6342813372612, 0.6659475564956665], "r_norm": [0.15152990818023682, 0.15035876631736755, 0.1519782394170761]}, {"step": 6500, "epoch": 13, "train_loss": 0.00011808508133981377, "cos_r_negg": [0.284700870513916, 0.3244197964668274, 0.36029553413391113], "cos_innovation_negg": [0.284700870513916, 0.3244197964668274, 0.36029553413391113], "cos_apical_negg": [0.002069351729005575, 0.04354419559240341, 0.08564615249633789], "cos_Ac_negg": [0.6112993359565735, 0.6342726349830627, 0.6662527322769165], "r_norm": [0.14418646693229675, 0.143080472946167, 0.14480093121528625], "ldr": {"dL_sdil": -0.00850454717874527, "dL_bp": -0.01731761544942856, "ratio": 0.49109227558381313}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9752, "test_loss": 0.13994757270812988}, {"step": 6600, "epoch": 14, "train_loss": 0.0002561457804404199, "cos_r_negg": [0.28083300590515137, 0.3208739459514618, 0.3552173972129822], "cos_innovation_negg": [0.28083300590515137, 0.3208739459514618, 0.3552173972129822], "cos_apical_negg": [0.0004630244802683592, 0.04371214285492897, 0.08422216773033142], "cos_Ac_negg": [0.6097625494003296, 0.6335102319717407, 0.6659466028213501], "r_norm": [0.14630337059497833, 0.14519445598125458, 0.14684632420539856]}, {"step": 6700, "epoch": 14, "train_loss": 0.0006219938513822854, "cos_r_negg": [0.2804514765739441, 0.3197934031486511, 0.354887455701828], "cos_innovation_negg": [0.2804514765739441, 0.3197934031486511, 0.354887455701828], "cos_apical_negg": [0.0014164652675390244, 0.04353693500161171, 0.08363394439220428], "cos_Ac_negg": [0.6096209287643433, 0.6323015689849854, 0.6662194132804871], "r_norm": [0.1471884846687317, 0.1460249125957489, 0.1477433294057846]}, {"step": 6800, "epoch": 14, "train_loss": 0.001066845841705799, "cos_r_negg": [0.26744145154953003, 0.30664917826652527, 0.3402431309223175], "cos_innovation_negg": [0.26744145154953003, 0.30664917826652527, 0.3402431309223175], "cos_apical_negg": [1.106318086385727e-05, 0.04225609451532364, 0.08050717413425446], "cos_Ac_negg": [0.6058058738708496, 0.6291998028755188, 0.663305401802063], "r_norm": [0.14706626534461975, 0.14594686031341553, 0.14794312417507172]}, {"step": 6900, "epoch": 14, "train_loss": 0.0008502452983520925, "cos_r_negg": [0.26708921790122986, 0.30680036544799805, 0.34202855825424194], "cos_innovation_negg": [0.26708921790122986, 0.30680036544799805, 0.34202855825424194], "cos_apical_negg": [0.00013900105841457844, 0.04170065373182297, 0.08219294995069504], "cos_Ac_negg": [0.6062815189361572, 0.629963755607605, 0.6647016406059265], "r_norm": [0.14870262145996094, 0.147597074508667, 0.149154931306839]}, {"step": 7000, "epoch": 14, "train_loss": 0.0005537670804187655, "cos_r_negg": [0.26967987418174744, 0.30989396572113037, 0.3444839119911194], "cos_innovation_negg": [0.26967987418174744, 0.30989396572113037, 0.3444839119911194], "cos_apical_negg": [-0.0012382407439872622, 0.04133274778723717, 0.08160696923732758], "cos_Ac_negg": [0.6068159341812134, 0.6314824819564819, 0.6662921905517578], "r_norm": [0.1491071879863739, 0.14798906445503235, 0.14975789189338684], "ldr": {"dL_sdil": -0.009437493979930878, "dL_bp": -0.019233211874961853, "ratio": 0.4906873610754936}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9748, "test_loss": 0.14247735996246338}], "final": {"test_acc": 0.9748, "test_loss": 0.14247735996246338, "wall_s": 25.2902090549469, "cos_r_negg": [0.26692265272140503, 0.30770188570022583, 0.3412211537361145], "cos_innovation_negg": [0.26692265272140503, 0.30770188570022583, 0.3412211537361145], "cos_apical_negg": [-0.0005659900489263237, 0.04179733619093895, 0.08092036843299866], "cos_Ac_negg": [0.6041395664215088, 0.6299995183944702, 0.6659080386161804]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_residual_s2.json b/results/nuis_ctrl_v1_rho0p05_residual_s2.json new file mode 100644 index 0000000..f15d82e --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_residual_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_residual_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.017675090581178665, 0.0372890904545784, 0.0025144016835838556], "cos_innovation_negg": [0.017675090581178665, 0.0372890904545784, 0.0025144016835838556], "cos_apical_negg": [0.01725875586271286, 0.0366811528801918, 0.002047688001766801], "cos_Ac_negg": [0.017675412818789482, 0.03728915750980377, 0.0025144696701318026], "r_norm": [4.74981689453125, 4.774450302124023, 4.856009483337402], "ldr": {"dL_sdil": -0.041077613830566406, "dL_bp": -0.24913907051086426, "ratio": 0.16487824951074917}}, {"step": 100, "epoch": 0, "train_loss": 0.17586416006088257, "cos_r_negg": [0.4970792233943939, 0.40330982208251953, 0.36845076084136963], "cos_innovation_negg": [0.4970792233943939, 0.40330982208251953, 0.36845076084136963], "cos_apical_negg": [0.19017258286476135, 0.17915573716163635, 0.21979236602783203], "cos_Ac_negg": [0.4971027076244354, 0.4034501314163208, 0.3684612214565277], "r_norm": [0.8698434829711914, 0.8818600177764893, 0.8946853876113892]}, {"step": 200, "epoch": 0, "train_loss": 0.3208368420600891, "cos_r_negg": [0.4926418364048004, 0.4460785388946533, 0.40469175577163696], "cos_innovation_negg": [0.4926418364048004, 0.4460785388946533, 0.40469175577163696], "cos_apical_negg": [0.19864384829998016, 0.190121129155159, 0.2166076898574829], "cos_Ac_negg": [0.492310106754303, 0.44555914402008057, 0.4045509696006775], "r_norm": [0.7164028882980347, 0.7233113050460815, 0.7400282025337219]}, {"step": 300, "epoch": 0, "train_loss": 0.19444385170936584, "cos_r_negg": [0.5353859663009644, 0.46626943349838257, 0.4177759885787964], "cos_innovation_negg": [0.5353859663009644, 0.46626943349838257, 0.4177759885787964], "cos_apical_negg": [0.16973918676376343, 0.1737947165966034, 0.2042451798915863], "cos_Ac_negg": [0.5351248383522034, 0.4660660922527313, 0.4175689220428467], "r_norm": [0.503006637096405, 0.5082999467849731, 0.5159482359886169]}, {"step": 400, "epoch": 0, "train_loss": 0.14225107431411743, "cos_r_negg": [0.5365469455718994, 0.5072477459907532, 0.4304060935974121], "cos_innovation_negg": [0.5365469455718994, 0.5072477459907532, 0.4304060935974121], "cos_apical_negg": [0.15741007030010223, 0.18071222305297852, 0.1913570910692215], "cos_Ac_negg": [0.5363162755966187, 0.5070614814758301, 0.4302181899547577], "r_norm": [0.5037429928779602, 0.5096166133880615, 0.5165624618530273]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9481, "test_loss": 0.17114948806762695}, {"step": 500, "epoch": 1, "train_loss": 0.20640233159065247, "cos_r_negg": [0.5260323286056519, 0.508039116859436, 0.46667802333831787], "cos_innovation_negg": [0.5260323286056519, 0.508039116859436, 0.46667802333831787], "cos_apical_negg": [0.14044275879859924, 0.1553192436695099, 0.19427356123924255], "cos_Ac_negg": [0.525843620300293, 0.5080826282501221, 0.46643710136413574], "r_norm": [0.473062664270401, 0.4806978702545166, 0.48895666003227234], "ldr": {"dL_sdil": -0.012392014265060425, "dL_bp": -0.010644689202308655, "ratio": 1.1641499370759274}}, {"step": 600, "epoch": 1, "train_loss": 0.10871553421020508, "cos_r_negg": [0.5452223420143127, 0.523560643196106, 0.45618343353271484], "cos_innovation_negg": [0.5452223420143127, 0.523560643196106, 0.45618343353271484], "cos_apical_negg": [0.1570211797952652, 0.15463560819625854, 0.17270517349243164], "cos_Ac_negg": [0.5453288555145264, 0.5229302644729614, 0.4559338092803955], "r_norm": [0.3991668224334717, 0.40303510427474976, 0.4114168882369995]}, {"step": 700, "epoch": 1, "train_loss": 0.17709866166114807, "cos_r_negg": [0.5708881616592407, 0.5400104522705078, 0.47234269976615906], "cos_innovation_negg": [0.5708881616592407, 0.5400104522705078, 0.47234269976615906], "cos_apical_negg": [0.14205503463745117, 0.15635958313941956, 0.18742695450782776], "cos_Ac_negg": [0.570570707321167, 0.539499044418335, 0.4720923900604248], "r_norm": [0.43761348724365234, 0.44108182191848755, 0.45164451003074646]}, {"step": 800, "epoch": 1, "train_loss": 0.15103770792484283, "cos_r_negg": [0.5683736801147461, 0.5581629276275635, 0.4943467378616333], "cos_innovation_negg": [0.5683736801147461, 0.5581629276275635, 0.4943467378616333], "cos_apical_negg": [0.13331329822540283, 0.15281610190868378, 0.1842871606349945], "cos_Ac_negg": [0.568236231803894, 0.5576614141464233, 0.49418404698371887], "r_norm": [0.33557310700416565, 0.3389970064163208, 0.34779369831085205]}, {"step": 900, "epoch": 1, "train_loss": 0.2185644805431366, "cos_r_negg": [0.5811727046966553, 0.5617357492446899, 0.48729169368743896], "cos_innovation_negg": [0.5811727046966553, 0.5617357492446899, 0.48729169368743896], "cos_apical_negg": [0.13030628859996796, 0.1538248360157013, 0.16992904245853424], "cos_Ac_negg": [0.5809875726699829, 0.5614639520645142, 0.4871039390563965], "r_norm": [0.38768357038497925, 0.39016199111938477, 0.39808550477027893]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9576, "test_loss": 0.14079903106689454}, {"step": 1000, "epoch": 2, "train_loss": 0.027318187057971954, "cos_r_negg": [0.5814144015312195, 0.5783625841140747, 0.4958611726760864], "cos_innovation_negg": [0.5814144015312195, 0.5783625841140747, 0.4958611726760864], "cos_apical_negg": [0.1055048257112503, 0.1402195692062378, 0.164465069770813], "cos_Ac_negg": [0.5812327265739441, 0.5781751275062561, 0.4956243634223938], "r_norm": [0.312904417514801, 0.31333309412002563, 0.32295629382133484], "ldr": {"dL_sdil": -0.0076730698347091675, "dL_bp": -0.006511993706226349, "ratio": 1.1782981036011555}}, {"step": 1100, "epoch": 2, "train_loss": 0.12655237317085266, "cos_r_negg": [0.5872601270675659, 0.5777523517608643, 0.5042988657951355], "cos_innovation_negg": [0.5872601270675659, 0.5777523517608643, 0.5042988657951355], "cos_apical_negg": [0.10918585956096649, 0.15159432590007782, 0.17792680859565735], "cos_Ac_negg": [0.588329553604126, 0.5776081681251526, 0.503897488117218], "r_norm": [0.35866644978523254, 0.3616602420806885, 0.37040048837661743]}, {"step": 1200, "epoch": 2, "train_loss": 0.19169515371322632, "cos_r_negg": [0.5778206586837769, 0.5714322328567505, 0.5065737366676331], "cos_innovation_negg": [0.5778206586837769, 0.5714322328567505, 0.5065737366676331], "cos_apical_negg": [0.103522390127182, 0.13620984554290771, 0.17392313480377197], "cos_Ac_negg": [0.5777586698532104, 0.571283757686615, 0.5059672594070435], "r_norm": [0.3244379758834839, 0.3290201425552368, 0.33428263664245605]}, {"step": 1300, "epoch": 2, "train_loss": 0.135965034365654, "cos_r_negg": [0.5901323556900024, 0.592176079750061, 0.5073062181472778], "cos_innovation_negg": [0.5901323556900024, 0.592176079750061, 0.5073062181472778], "cos_apical_negg": [0.09399458765983582, 0.12740005552768707, 0.1656610667705536], "cos_Ac_negg": [0.5904358625411987, 0.5921123027801514, 0.5068682432174683], "r_norm": [0.3208772540092468, 0.3238424062728882, 0.32930833101272583]}, {"step": 1400, "epoch": 2, "train_loss": 0.1040671095252037, "cos_r_negg": [0.5833787322044373, 0.5839993953704834, 0.5164324045181274], "cos_innovation_negg": [0.5833787322044373, 0.5839993953704834, 0.5164324045181274], "cos_apical_negg": [0.08873105049133301, 0.10539083927869797, 0.15748608112335205], "cos_Ac_negg": [0.5840877294540405, 0.584343433380127, 0.5154862403869629], "r_norm": [0.26289546489715576, 0.2639598548412323, 0.2716895341873169]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9691, "test_loss": 0.10747614059448242}, {"step": 1500, "epoch": 3, "train_loss": 0.062176160514354706, "cos_r_negg": [0.5943048000335693, 0.5952531099319458, 0.5310481786727905], "cos_innovation_negg": [0.5943048000335693, 0.5952531099319458, 0.5310481786727905], "cos_apical_negg": [0.08299660682678223, 0.10385184735059738, 0.15107882022857666], "cos_Ac_negg": [0.5947880148887634, 0.5951738357543945, 0.5299400687217712], "r_norm": [0.27155426144599915, 0.2732366621494293, 0.27872419357299805], "ldr": {"dL_sdil": -0.007850125432014465, "dL_bp": -0.007701881229877472, "ratio": 1.0192477912489637}}, {"step": 1600, "epoch": 3, "train_loss": 0.04608044773340225, "cos_r_negg": [0.596003532409668, 0.6089153289794922, 0.5469405055046082], "cos_innovation_negg": [0.596003532409668, 0.6089153289794922, 0.5469405055046082], "cos_apical_negg": [0.08411060273647308, 0.10125107318162918, 0.15174731612205505], "cos_Ac_negg": [0.5981305241584778, 0.6094847917556763, 0.5464131832122803], "r_norm": [0.2744586169719696, 0.27718842029571533, 0.28355464339256287]}, {"step": 1700, "epoch": 3, "train_loss": 0.042360708117485046, "cos_r_negg": [0.599576473236084, 0.6094846129417419, 0.5494703054428101], "cos_innovation_negg": [0.599576473236084, 0.6094846129417419, 0.5494703054428101], "cos_apical_negg": [0.08860689401626587, 0.10659298300743103, 0.15702548623085022], "cos_Ac_negg": [0.6054372191429138, 0.6122726798057556, 0.5488888025283813], "r_norm": [0.29002389311790466, 0.2930294871330261, 0.30011770129203796]}, {"step": 1800, "epoch": 3, "train_loss": 0.04638168588280678, "cos_r_negg": [0.5836105942726135, 0.6037336587905884, 0.5602104663848877], "cos_innovation_negg": [0.5836105942726135, 0.6037336587905884, 0.5602104663848877], "cos_apical_negg": [0.048589885234832764, 0.09653361141681671, 0.15400061011314392], "cos_Ac_negg": [0.5913264155387878, 0.6077687740325928, 0.5613003969192505], "r_norm": [0.23215599358081818, 0.23303121328353882, 0.2408786416053772]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9667, "test_loss": 0.11098653583526612}, {"step": 1900, "epoch": 4, "train_loss": 0.030510950833559036, "cos_r_negg": [0.5977516174316406, 0.6129684448242188, 0.5715459585189819], "cos_innovation_negg": [0.5977516174316406, 0.6129684448242188, 0.5715459585189819], "cos_apical_negg": [0.07869121432304382, 0.11038168519735336, 0.15888436138629913], "cos_Ac_negg": [0.6013244390487671, 0.6158010959625244, 0.5713731646537781], "r_norm": [0.2544582784175873, 0.2550382614135742, 0.2616143226623535]}, {"step": 2000, "epoch": 4, "train_loss": 0.06422892212867737, "cos_r_negg": [0.5751674175262451, 0.6050205230712891, 0.5632307529449463], "cos_innovation_negg": [0.5751674175262451, 0.6050205230712891, 0.5632307529449463], "cos_apical_negg": [0.039523810148239136, 0.08830837160348892, 0.13610044121742249], "cos_Ac_negg": [0.5866235494613647, 0.6144043207168579, 0.5659879446029663], "r_norm": [0.20273345708847046, 0.2024599313735962, 0.20888051390647888], "ldr": {"dL_sdil": -0.004440866410732269, "dL_bp": -0.004471570253372192, "ratio": 0.9931335434981106}}, {"step": 2100, "epoch": 4, "train_loss": 0.02168159931898117, "cos_r_negg": [0.5857771635055542, 0.6053357124328613, 0.5740090608596802], "cos_innovation_negg": [0.5857771635055542, 0.6053357124328613, 0.5740090608596802], "cos_apical_negg": [0.05207394063472748, 0.09032325446605682, 0.14031092822551727], "cos_Ac_negg": [0.5964084267616272, 0.6146030426025391, 0.5765020847320557], "r_norm": [0.2210220992565155, 0.2213449776172638, 0.22735489904880524]}, {"step": 2200, "epoch": 4, "train_loss": 0.009620489552617073, "cos_r_negg": [0.5972596406936646, 0.6263022422790527, 0.5853859186172485], "cos_innovation_negg": [0.5972596406936646, 0.6263022422790527, 0.5853859186172485], "cos_apical_negg": [0.061493098735809326, 0.10348585247993469, 0.1426779329776764], "cos_Ac_negg": [0.6023486852645874, 0.6299389600753784, 0.5839298963546753], "r_norm": [0.26552489399909973, 0.26564037799835205, 0.27185261249542236]}, {"step": 2300, "epoch": 4, "train_loss": 0.17019157111644745, "cos_r_negg": [0.5706629753112793, 0.6006418466567993, 0.5666520595550537], "cos_innovation_negg": [0.5706629753112793, 0.6006418466567993, 0.5666520595550537], "cos_apical_negg": [0.06424862891435623, 0.09687228500843048, 0.14904698729515076], "cos_Ac_negg": [0.5960767269134521, 0.6189320087432861, 0.580556333065033], "r_norm": [0.23334062099456787, 0.23265743255615234, 0.23751312494277954]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9695, "test_loss": 0.1124767749786377}, {"step": 2400, "epoch": 5, "train_loss": 0.06642969697713852, "cos_r_negg": [0.5909543633460999, 0.6104036569595337, 0.5869391560554504], "cos_innovation_negg": [0.5909543633460999, 0.6104036569595337, 0.5869391560554504], "cos_apical_negg": [0.04040750488638878, 0.06977728009223938, 0.13496091961860657], "cos_Ac_negg": [0.6188592910766602, 0.6329231262207031, 0.5975795984268188], "r_norm": [0.1879744976758957, 0.18929597735404968, 0.19415967166423798]}, {"step": 2500, "epoch": 5, "train_loss": 0.01560700498521328, "cos_r_negg": [0.5764260292053223, 0.6069477796554565, 0.578482985496521], "cos_innovation_negg": [0.5764260292053223, 0.6069477796554565, 0.578482985496521], "cos_apical_negg": [0.029892697930336, 0.08527258038520813, 0.14372044801712036], "cos_Ac_negg": [0.6116330027580261, 0.6322923898696899, 0.5927160978317261], "r_norm": [0.24426546692848206, 0.24442210793495178, 0.24917516112327576], "ldr": {"dL_sdil": -0.01148965209722519, "dL_bp": -0.012451238930225372, "ratio": 0.9227717949684564}}, {"step": 2600, "epoch": 5, "train_loss": 0.05091261863708496, "cos_r_negg": [0.5831518173217773, 0.6210675239562988, 0.5870150327682495], "cos_innovation_negg": [0.5831518173217773, 0.6210675239562988, 0.5870150327682495], "cos_apical_negg": [0.04560829699039459, 0.09175842255353928, 0.14557278156280518], "cos_Ac_negg": [0.6158068776130676, 0.6439459919929504, 0.5956686735153198], "r_norm": [0.23803922533988953, 0.23771339654922485, 0.24180249869823456]}, {"step": 2700, "epoch": 5, "train_loss": 0.04318687692284584, "cos_r_negg": [0.5957183837890625, 0.6321381330490112, 0.5912017226219177], "cos_innovation_negg": [0.5957183837890625, 0.6321381330490112, 0.5912017226219177], "cos_apical_negg": [0.03164352849125862, 0.08044534176588058, 0.13389158248901367], "cos_Ac_negg": [0.6257493495941162, 0.6570411324501038, 0.602735161781311], "r_norm": [0.19935742020606995, 0.19892239570617676, 0.2033696472644806]}, {"step": 2800, "epoch": 5, "train_loss": 0.08792144060134888, "cos_r_negg": [0.5771921873092651, 0.6217223405838013, 0.5921776294708252], "cos_innovation_negg": [0.5771921873092651, 0.6217223405838013, 0.5921776294708252], "cos_apical_negg": [0.02564740926027298, 0.0745646134018898, 0.1367609202861786], "cos_Ac_negg": [0.6057387590408325, 0.6472271084785461, 0.6045480370521545], "r_norm": [0.18388976156711578, 0.1832829862833023, 0.18803295493125916]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9703, "test_loss": 0.10882798805236817}, {"step": 2900, "epoch": 6, "train_loss": 0.013013310730457306, "cos_r_negg": [0.5745726823806763, 0.6178932785987854, 0.5860469341278076], "cos_innovation_negg": [0.5745726823806763, 0.6178932785987854, 0.5860469341278076], "cos_apical_negg": [0.024404019117355347, 0.08036196231842041, 0.14160391688346863], "cos_Ac_negg": [0.6131798028945923, 0.6507341861724854, 0.6046215295791626], "r_norm": [0.20108121633529663, 0.20018455386161804, 0.20396503806114197]}, {"step": 3000, "epoch": 6, "train_loss": 0.033054664731025696, "cos_r_negg": [0.562021017074585, 0.6133217215538025, 0.5866560935974121], "cos_innovation_negg": [0.562021017074585, 0.6133217215538025, 0.5866560935974121], "cos_apical_negg": [0.013995595276355743, 0.06926731765270233, 0.12820187211036682], "cos_Ac_negg": [0.6141505241394043, 0.6563743352890015, 0.6088958978652954], "r_norm": [0.15822497010231018, 0.15869250893592834, 0.16342765092849731], "ldr": {"dL_sdil": -0.005045607686042786, "dL_bp": -0.006118357181549072, "ratio": 0.8246670693333593}}, {"step": 3100, "epoch": 6, "train_loss": 0.03830147534608841, "cos_r_negg": [0.5494112372398376, 0.6081170439720154, 0.5899116396903992], "cos_innovation_negg": [0.5494112372398376, 0.6081170439720154, 0.5899116396903992], "cos_apical_negg": [0.030363958328962326, 0.08567628264427185, 0.13646987080574036], "cos_Ac_negg": [0.6140666007995605, 0.6589256525039673, 0.6149048805236816], "r_norm": [0.1938188672065735, 0.1934228241443634, 0.1997278928756714]}, {"step": 3200, "epoch": 6, "train_loss": 0.04877293109893799, "cos_r_negg": [0.5403801202774048, 0.5943267941474915, 0.5810519456863403], "cos_innovation_negg": [0.5403801202774048, 0.5943267941474915, 0.5810519456863403], "cos_apical_negg": [0.015142959542572498, 0.07472680509090424, 0.1282399594783783], "cos_Ac_negg": [0.6186503171920776, 0.6629937887191772, 0.6203477382659912], "r_norm": [0.1796012818813324, 0.17909282445907593, 0.18256071209907532]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9721, "test_loss": 0.10792610893249512}, {"step": 3300, "epoch": 7, "train_loss": 0.009267272427678108, "cos_r_negg": [0.5438717007637024, 0.5952125787734985, 0.5815542936325073], "cos_innovation_negg": [0.5438717007637024, 0.5952125787734985, 0.5815542936325073], "cos_apical_negg": [0.026580266654491425, 0.07269361615180969, 0.1270410120487213], "cos_Ac_negg": [0.6219054460525513, 0.6613340377807617, 0.6265897154808044], "r_norm": [0.16144952178001404, 0.1611819714307785, 0.16384157538414001]}, {"step": 3400, "epoch": 7, "train_loss": 0.01776883751153946, "cos_r_negg": [0.5103996992111206, 0.5722621083259583, 0.5747686624526978], "cos_innovation_negg": [0.5103996992111206, 0.5722621083259583, 0.5747686624526978], "cos_apical_negg": [0.023606907576322556, 0.07380744814872742, 0.13563251495361328], "cos_Ac_negg": [0.6165734529495239, 0.6589145660400391, 0.6254906058311462], "r_norm": [0.18773292005062103, 0.1874973475933075, 0.19095928966999054]}, {"step": 3500, "epoch": 7, "train_loss": 0.00803975947201252, "cos_r_negg": [0.5079407691955566, 0.5599184036254883, 0.5622813701629639], "cos_innovation_negg": [0.5079407691955566, 0.5599184036254883, 0.5622813701629639], "cos_apical_negg": [0.009438520297408104, 0.0698479413986206, 0.12053480744361877], "cos_Ac_negg": [0.6317951679229736, 0.6660887002944946, 0.636048436164856], "r_norm": [0.16335417330265045, 0.1640072762966156, 0.16649848222732544], "ldr": {"dL_sdil": -0.0055382028222084045, "dL_bp": -0.007310442626476288, "ratio": 0.7575742133794542}}, {"step": 3600, "epoch": 7, "train_loss": 0.0013797478750348091, "cos_r_negg": [0.5053645968437195, 0.5602267384529114, 0.5643929839134216], "cos_innovation_negg": [0.5053645968437195, 0.5602267384529114, 0.5643929839134216], "cos_apical_negg": [0.010374944657087326, 0.07207032293081284, 0.12697625160217285], "cos_Ac_negg": [0.6321369409561157, 0.6641747951507568, 0.6335896253585815], "r_norm": [0.1691068708896637, 0.1682838499546051, 0.17226824164390564]}, {"step": 3700, "epoch": 7, "train_loss": 0.0032279654406011105, "cos_r_negg": [0.5042350888252258, 0.5589630603790283, 0.5636758208274841], "cos_innovation_negg": [0.5042350888252258, 0.5589630603790283, 0.5636758208274841], "cos_apical_negg": [0.019520625472068787, 0.07704445719718933, 0.1362655609846115], "cos_Ac_negg": [0.6336003541946411, 0.6712273359298706, 0.6419575214385986], "r_norm": [0.18173480033874512, 0.18056946992874146, 0.18452316522598267]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9733, "test_loss": 0.11650427446365356}, {"step": 3800, "epoch": 8, "train_loss": 0.00982544757425785, "cos_r_negg": [0.46905720233917236, 0.5218753218650818, 0.5423223972320557], "cos_innovation_negg": [0.46905720233917236, 0.5218753218650818, 0.5423223972320557], "cos_apical_negg": [0.019694751128554344, 0.06497429311275482, 0.13053977489471436], "cos_Ac_negg": [0.6314952373504639, 0.666953444480896, 0.6417737603187561], "r_norm": [0.16658152639865875, 0.16555774211883545, 0.16968370974063873]}, {"step": 3900, "epoch": 8, "train_loss": 0.02925984561443329, "cos_r_negg": [0.4413347542285919, 0.49921470880508423, 0.5250418186187744], "cos_innovation_negg": [0.4413347542285919, 0.49921470880508423, 0.5250418186187744], "cos_apical_negg": [0.011488229967653751, 0.06818787753582001, 0.12613055109977722], "cos_Ac_negg": [0.6245931386947632, 0.6577432155609131, 0.6453928351402283], "r_norm": [0.15324461460113525, 0.15281051397323608, 0.15677839517593384]}, {"step": 4000, "epoch": 8, "train_loss": 0.008045164868235588, "cos_r_negg": [0.42673054337501526, 0.49403172731399536, 0.5183005332946777], "cos_innovation_negg": [0.42673054337501526, 0.49403172731399536, 0.5183005332946777], "cos_apical_negg": [0.0051508452743291855, 0.06453575938940048, 0.12304888665676117], "cos_Ac_negg": [0.6234814524650574, 0.6618207693099976, 0.6484739780426025], "r_norm": [0.17489200830459595, 0.17475000023841858, 0.17803341150283813], "ldr": {"dL_sdil": -0.006972499191761017, "dL_bp": -0.010512404143810272, "ratio": 0.6632639971196732}}, {"step": 4100, "epoch": 8, "train_loss": 0.0034962778445333242, "cos_r_negg": [0.4441763758659363, 0.5131612420082092, 0.5346510410308838], "cos_innovation_negg": [0.4441763758659363, 0.5131612420082092, 0.5346510410308838], "cos_apical_negg": [0.009656371548771858, 0.06788153201341629, 0.12598329782485962], "cos_Ac_negg": [0.6313721537590027, 0.6735833883285522, 0.6593194007873535], "r_norm": [0.1783696860074997, 0.1777813732624054, 0.18104714155197144]}, {"step": 4200, "epoch": 8, "train_loss": 0.008683789521455765, "cos_r_negg": [0.44431424140930176, 0.5114341974258423, 0.5370839834213257], "cos_innovation_negg": [0.44431424140930176, 0.5114341974258423, 0.5370839834213257], "cos_apical_negg": [0.009143955074250698, 0.0695948675274849, 0.12227039039134979], "cos_Ac_negg": [0.6305049061775208, 0.6718047857284546, 0.6562168598175049], "r_norm": [0.16913482546806335, 0.16838935017585754, 0.17084577679634094]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9739, "test_loss": 0.1178732195854187}, {"step": 4300, "epoch": 9, "train_loss": 0.0016776140546426177, "cos_r_negg": [0.43030187487602234, 0.4954873323440552, 0.5225053429603577], "cos_innovation_negg": [0.43030187487602234, 0.4954873323440552, 0.5225053429603577], "cos_apical_negg": [0.011808861047029495, 0.06989718228578568, 0.11824727803468704], "cos_Ac_negg": [0.6243135333061218, 0.6735286116600037, 0.6625051498413086], "r_norm": [0.16774168610572815, 0.16727542877197266, 0.16974994540214539]}, {"step": 4400, "epoch": 9, "train_loss": 0.0011276163859292865, "cos_r_negg": [0.3988119959831238, 0.4685785174369812, 0.4951285719871521], "cos_innovation_negg": [0.3988119959831238, 0.4685785174369812, 0.4951285719871521], "cos_apical_negg": [0.006170487031340599, 0.06447477638721466, 0.11312360316514969], "cos_Ac_negg": [0.6207109689712524, 0.6658036112785339, 0.6566041111946106], "r_norm": [0.16017498075962067, 0.15935488045215607, 0.1622387021780014]}, {"step": 4500, "epoch": 9, "train_loss": 0.08726860582828522, "cos_r_negg": [0.38311997056007385, 0.44503679871559143, 0.4763762950897217], "cos_innovation_negg": [0.38311997056007385, 0.44503679871559143, 0.4763762950897217], "cos_apical_negg": [0.00993735808879137, 0.06415434181690216, 0.11535833775997162], "cos_Ac_negg": [0.619747519493103, 0.6640850305557251, 0.6565393209457397], "r_norm": [0.1903897374868393, 0.18939487636089325, 0.19168758392333984], "ldr": {"dL_sdil": -0.012501165270805359, "dL_bp": -0.019839875400066376, "ratio": 0.6301030131854324}}, {"step": 4600, "epoch": 9, "train_loss": 0.0018958705477416515, "cos_r_negg": [0.3746110200881958, 0.44570279121398926, 0.4720151126384735], "cos_innovation_negg": [0.3746110200881958, 0.44570279121398926, 0.4720151126384735], "cos_apical_negg": [0.005030651111155748, 0.06676638126373291, 0.1082020178437233], "cos_Ac_negg": [0.6145462393760681, 0.6667003631591797, 0.6569769382476807], "r_norm": [0.1803533285856247, 0.17894554138183594, 0.18128520250320435]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.975, "test_loss": 0.11868392906188965}, {"step": 4700, "epoch": 10, "train_loss": 0.0014905092539265752, "cos_r_negg": [0.35386329889297485, 0.41931837797164917, 0.4501698613166809], "cos_innovation_negg": [0.35386329889297485, 0.41931837797164917, 0.4501698613166809], "cos_apical_negg": [0.005821586586534977, 0.061907459050416946, 0.10470347106456757], "cos_Ac_negg": [0.6195168495178223, 0.6647994518280029, 0.6555296182632446], "r_norm": [0.16766610741615295, 0.16651062667369843, 0.17018306255340576]}, {"step": 4800, "epoch": 10, "train_loss": 0.0012656109174713492, "cos_r_negg": [0.34898343682289124, 0.41529911756515503, 0.4481699466705322], "cos_innovation_negg": [0.34898343682289124, 0.41529911756515503, 0.4481699466705322], "cos_apical_negg": [0.005106424912810326, 0.06552715599536896, 0.10781456530094147], "cos_Ac_negg": [0.6147809028625488, 0.66548091173172, 0.6564418077468872], "r_norm": [0.1745816171169281, 0.17323486506938934, 0.1773829162120819]}, {"step": 4900, "epoch": 10, "train_loss": 0.0012779165990650654, "cos_r_negg": [0.3501048684120178, 0.4089609980583191, 0.4419053792953491], "cos_innovation_negg": [0.3501048684120178, 0.4089609980583191, 0.4419053792953491], "cos_apical_negg": [0.006679007317870855, 0.06383780390024185, 0.10522712767124176], "cos_Ac_negg": [0.6193428039550781, 0.6660074591636658, 0.6599264144897461], "r_norm": [0.17439880967140198, 0.17352086305618286, 0.17672832310199738]}, {"step": 5000, "epoch": 10, "train_loss": 0.000160726165631786, "cos_r_negg": [0.33845794200897217, 0.3988485336303711, 0.4336903691291809], "cos_innovation_negg": [0.33845794200897217, 0.3988485336303711, 0.4336903691291809], "cos_apical_negg": [0.0021523749455809593, 0.06200505793094635, 0.10135708749294281], "cos_Ac_negg": [0.614773154258728, 0.6642279028892517, 0.6590754985809326], "r_norm": [0.17232462763786316, 0.17092199623584747, 0.1742478609085083], "ldr": {"dL_sdil": -0.009563423693180084, "dL_bp": -0.01564066857099533, "ratio": 0.6114459653543757}}, {"step": 5100, "epoch": 10, "train_loss": 0.0022132722660899162, "cos_r_negg": [0.3401888608932495, 0.4020995497703552, 0.4394442141056061], "cos_innovation_negg": [0.3401888608932495, 0.4020995497703552, 0.4394442141056061], "cos_apical_negg": [0.0012293346226215363, 0.06330874562263489, 0.10881869494915009], "cos_Ac_negg": [0.6191054582595825, 0.668574869632721, 0.6650499105453491], "r_norm": [0.17797641456127167, 0.17698729038238525, 0.1799686849117279]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9754, "test_loss": 0.1257467162132263}, {"step": 5200, "epoch": 11, "train_loss": 0.0004062363877892494, "cos_r_negg": [0.3309895098209381, 0.3949207067489624, 0.4280523359775543], "cos_innovation_negg": [0.3309895098209381, 0.3949207067489624, 0.4280523359775543], "cos_apical_negg": [-0.0009264005348086357, 0.06143319979310036, 0.10022175312042236], "cos_Ac_negg": [0.6125749349594116, 0.6660000085830688, 0.6617434024810791], "r_norm": [0.16346126794815063, 0.16259120404720306, 0.1651577651500702]}, {"step": 5300, "epoch": 11, "train_loss": 0.0014113449724391103, "cos_r_negg": [0.3236422538757324, 0.38755232095718384, 0.41874659061431885], "cos_innovation_negg": [0.3236422538757324, 0.38755232095718384, 0.41874659061431885], "cos_apical_negg": [0.0011327746324241161, 0.061301615089178085, 0.097899429500103], "cos_Ac_negg": [0.6073694229125977, 0.6618682146072388, 0.661133885383606], "r_norm": [0.1701488494873047, 0.16920945048332214, 0.1716749519109726]}, {"step": 5400, "epoch": 11, "train_loss": 0.0017204724717885256, "cos_r_negg": [0.32189029455184937, 0.3826256990432739, 0.4150451719760895], "cos_innovation_negg": [0.32189029455184937, 0.3826256990432739, 0.4150451719760895], "cos_apical_negg": [-0.001213527750223875, 0.060469381511211395, 0.09835895895957947], "cos_Ac_negg": [0.6155924201011658, 0.6649582386016846, 0.6622479557991028], "r_norm": [0.16895024478435516, 0.16803152859210968, 0.17112287878990173]}, {"step": 5500, "epoch": 11, "train_loss": 0.000993720255792141, "cos_r_negg": [0.32139259576797485, 0.38371407985687256, 0.41844815015792847], "cos_innovation_negg": [0.32139259576797485, 0.38371407985687256, 0.41844815015792847], "cos_apical_negg": [-0.001972053200006485, 0.061453819274902344, 0.09971621632575989], "cos_Ac_negg": [0.6149802207946777, 0.6682905554771423, 0.665856122970581], "r_norm": [0.17345094680786133, 0.17252591252326965, 0.17564040422439575], "ldr": {"dL_sdil": -0.009622596204280853, "dL_bp": -0.016050435602664948, "ratio": 0.5995224330661255}}, {"step": 5600, "epoch": 11, "train_loss": 0.0016657065134495497, "cos_r_negg": [0.30397194623947144, 0.36559394001960754, 0.39393600821495056], "cos_innovation_negg": [0.30397194623947144, 0.36559394001960754, 0.39393600821495056], "cos_apical_negg": [-0.003711450845003128, 0.05918329954147339, 0.08927440643310547], "cos_Ac_negg": [0.6115577220916748, 0.6611321568489075, 0.6605039834976196], "r_norm": [0.1635160744190216, 0.16261613368988037, 0.16581760346889496]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9757, "test_loss": 0.12609906377792357}, {"step": 5700, "epoch": 12, "train_loss": 0.0013611136237159371, "cos_r_negg": [0.3041272759437561, 0.3620166778564453, 0.39632081985473633], "cos_innovation_negg": [0.3041272759437561, 0.3620166778564453, 0.39632081985473633], "cos_apical_negg": [-0.0015211999416351318, 0.05659618228673935, 0.09308314323425293], "cos_Ac_negg": [0.6126233339309692, 0.665428102016449, 0.6645920276641846], "r_norm": [0.1696382761001587, 0.16894446313381195, 0.17202845215797424]}, {"step": 5800, "epoch": 12, "train_loss": 0.0020447673741728067, "cos_r_negg": [0.2991286516189575, 0.35727763175964355, 0.3913206160068512], "cos_innovation_negg": [0.2991286516189575, 0.35727763175964355, 0.3913206160068512], "cos_apical_negg": [-0.003953907638788223, 0.057026591151952744, 0.0929592177271843], "cos_Ac_negg": [0.6113331913948059, 0.6634926795959473, 0.663231372833252], "r_norm": [0.1672661155462265, 0.16649901866912842, 0.16933535039424896]}, {"step": 5900, "epoch": 12, "train_loss": 0.0014066663570702076, "cos_r_negg": [0.30047860741615295, 0.36044222116470337, 0.39695295691490173], "cos_innovation_negg": [0.30047860741615295, 0.36044222116470337, 0.39695295691490173], "cos_apical_negg": [0.0013358844444155693, 0.05819650739431381, 0.09572765231132507], "cos_Ac_negg": [0.6117909550666809, 0.6672633290290833, 0.6678286194801331], "r_norm": [0.1781907081604004, 0.17731031775474548, 0.17990857362747192]}, {"step": 6000, "epoch": 12, "train_loss": 0.0002284040383528918, "cos_r_negg": [0.2873755693435669, 0.34967178106307983, 0.38316577672958374], "cos_innovation_negg": [0.2873755693435669, 0.34967178106307983, 0.38316577672958374], "cos_apical_negg": [-0.0016045423690229654, 0.05784754827618599, 0.09123990684747696], "cos_Ac_negg": [0.6095637083053589, 0.6628272533416748, 0.6649200916290283], "r_norm": [0.17103305459022522, 0.17006808519363403, 0.1730501502752304], "ldr": {"dL_sdil": -0.009525716304779053, "dL_bp": -0.01639055460691452, "ratio": 0.5811710789066609}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9758, "test_loss": 0.12974705410003662}, {"step": 6100, "epoch": 13, "train_loss": 0.0007630693726241589, "cos_r_negg": [0.2865099608898163, 0.35144031047821045, 0.3844558000564575], "cos_innovation_negg": [0.2865099608898163, 0.35144031047821045, 0.3844558000564575], "cos_apical_negg": [-0.004450838081538677, 0.05715659260749817, 0.09215658903121948], "cos_Ac_negg": [0.6112799644470215, 0.6659401655197144, 0.6665726900100708], "r_norm": [0.1627330332994461, 0.161863774061203, 0.1650581657886505]}, {"step": 6200, "epoch": 13, "train_loss": 0.0005757695762440562, "cos_r_negg": [0.2834436893463135, 0.3464440703392029, 0.37994325160980225], "cos_innovation_negg": [0.2834436893463135, 0.3464440703392029, 0.37994325160980225], "cos_apical_negg": [-0.005105349700897932, 0.05714293196797371, 0.09181632101535797], "cos_Ac_negg": [0.6104350090026855, 0.663049042224884, 0.6663385629653931], "r_norm": [0.1573629379272461, 0.15636897087097168, 0.15965387225151062]}, {"step": 6300, "epoch": 13, "train_loss": 0.0006871781661175191, "cos_r_negg": [0.27674785256385803, 0.33835023641586304, 0.3720698058605194], "cos_innovation_negg": [0.27674785256385803, 0.33835023641586304, 0.3720698058605194], "cos_apical_negg": [-0.006839130073785782, 0.05593195557594299, 0.08954693377017975], "cos_Ac_negg": [0.6090211868286133, 0.660690426826477, 0.6654565930366516], "r_norm": [0.15734589099884033, 0.15652035176753998, 0.159844771027565]}, {"step": 6400, "epoch": 13, "train_loss": 0.0004738799179904163, "cos_r_negg": [0.2787921130657196, 0.3402276933193207, 0.3789054751396179], "cos_innovation_negg": [0.2787921130657196, 0.3402276933193207, 0.3789054751396179], "cos_apical_negg": [-0.005164789501577616, 0.05688251182436943, 0.09416162222623825], "cos_Ac_negg": [0.6102557182312012, 0.6640920639038086, 0.668114960193634], "r_norm": [0.16442690789699554, 0.1634214222431183, 0.16653040051460266]}, {"step": 6500, "epoch": 13, "train_loss": 0.000710058375261724, "cos_r_negg": [0.2747942805290222, 0.33808666467666626, 0.37204474210739136], "cos_innovation_negg": [0.2747942805290222, 0.33808666467666626, 0.37204474210739136], "cos_apical_negg": [-0.00583961745724082, 0.0555395670235157, 0.08962330967187881], "cos_Ac_negg": [0.6093389987945557, 0.6635269522666931, 0.6665266156196594], "r_norm": [0.16985037922859192, 0.16904151439666748, 0.1723356395959854], "ldr": {"dL_sdil": -0.009799778461456299, "dL_bp": -0.017330914735794067, "ratio": 0.5654507341852255}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9762, "test_loss": 0.13262854080200195}, {"step": 6600, "epoch": 14, "train_loss": 0.00024427290190942585, "cos_r_negg": [0.27022114396095276, 0.33344656229019165, 0.36703813076019287], "cos_innovation_negg": [0.27022114396095276, 0.33344656229019165, 0.36703813076019287], "cos_apical_negg": [-0.0057905614376068115, 0.056866422295570374, 0.08863642811775208], "cos_Ac_negg": [0.6120237112045288, 0.6635233163833618, 0.6657634973526001], "r_norm": [0.16316649317741394, 0.16228581964969635, 0.16553249955177307]}, {"step": 6700, "epoch": 14, "train_loss": 0.0004823197377845645, "cos_r_negg": [0.26939457654953003, 0.3327721655368805, 0.36749890446662903], "cos_innovation_negg": [0.26939457654953003, 0.3327721655368805, 0.36749890446662903], "cos_apical_negg": [-0.006343171000480652, 0.05564163625240326, 0.0884370505809784], "cos_Ac_negg": [0.6096881628036499, 0.6633120775222778, 0.6669067144393921], "r_norm": [0.16940899193286896, 0.16868823766708374, 0.1714802384376526]}, {"step": 6800, "epoch": 14, "train_loss": 0.0006483035394921899, "cos_r_negg": [0.2663818597793579, 0.3270002603530884, 0.360596239566803], "cos_innovation_negg": [0.2663818597793579, 0.3270002603530884, 0.360596239566803], "cos_apical_negg": [-0.005050738342106342, 0.05709920823574066, 0.08860805630683899], "cos_Ac_negg": [0.6111637949943542, 0.6632071733474731, 0.6660500764846802], "r_norm": [0.17243969440460205, 0.1715322583913803, 0.17472121119499207]}, {"step": 6900, "epoch": 14, "train_loss": 0.000526372401509434, "cos_r_negg": [0.2578943967819214, 0.3195595145225525, 0.3518974483013153], "cos_innovation_negg": [0.2578943967819214, 0.3195595145225525, 0.3518974483013153], "cos_apical_negg": [-0.006891245488077402, 0.056059278547763824, 0.08534833788871765], "cos_Ac_negg": [0.6097977161407471, 0.6608104705810547, 0.6642349362373352], "r_norm": [0.15979497134685516, 0.15897135436534882, 0.16206049919128418]}, {"step": 7000, "epoch": 14, "train_loss": 0.0003984446229878813, "cos_r_negg": [0.25704672932624817, 0.3179672658443451, 0.3501430153846741], "cos_innovation_negg": [0.25704672932624817, 0.3179672658443451, 0.3501430153846741], "cos_apical_negg": [-0.00590121652930975, 0.05605883151292801, 0.0864536315202713], "cos_Ac_negg": [0.6106324195861816, 0.6610426306724548, 0.6641703844070435], "r_norm": [0.15977753698825836, 0.15890425443649292, 0.16240063309669495], "ldr": {"dL_sdil": -0.008981242775917053, "dL_bp": -0.016131021082401276, "ratio": 0.55676839860531}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9759, "test_loss": 0.13539633264541626}], "final": {"test_acc": 0.9759, "test_loss": 0.13539633264541626, "wall_s": 23.59708571434021, "cos_r_negg": [0.2558465003967285, 0.3163191080093384, 0.34814485907554626], "cos_innovation_negg": [0.2558465003967285, 0.3163191080093384, 0.34814485907554626], "cos_apical_negg": [-0.0059834434650838375, 0.05518651381134987, 0.08496376872062683], "cos_Ac_negg": [0.6108644008636475, 0.6611005663871765, 0.6636170148849487]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_residual_s3.json b/results/nuis_ctrl_v1_rho0p05_residual_s3.json new file mode 100644 index 0000000..6fe4a48 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_residual_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_residual_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.014014249667525291, 0.01371424738317728, 0.006087127141654491], "cos_innovation_negg": [-0.014014249667525291, 0.01371424738317728, 0.006087127141654491], "cos_apical_negg": [-0.014671075157821178, 0.012926098890602589, 0.005224401131272316], "cos_Ac_negg": [-0.01401443500071764, 0.013714214786887169, 0.006087020039558411], "r_norm": [4.8543853759765625, 4.749162673950195, 4.848544120788574], "ldr": {"dL_sdil": -0.005753755569458008, "dL_bp": -0.23060989379882812, "ratio": 0.024950167899029}}, {"step": 100, "epoch": 0, "train_loss": 0.18896791338920593, "cos_r_negg": [0.4277474284172058, 0.36101263761520386, 0.3145836591720581], "cos_innovation_negg": [0.4277474284172058, 0.36101263761520386, 0.3145836591720581], "cos_apical_negg": [0.18812501430511475, 0.17243920266628265, 0.19488902390003204], "cos_Ac_negg": [0.42809146642684937, 0.3610365688800812, 0.3144207000732422], "r_norm": [0.861072301864624, 0.8483306169509888, 0.8574384450912476]}, {"step": 200, "epoch": 0, "train_loss": 0.2482275664806366, "cos_r_negg": [0.4761043190956116, 0.43292197585105896, 0.34609800577163696], "cos_innovation_negg": [0.4761043190956116, 0.43292197585105896, 0.34609800577163696], "cos_apical_negg": [0.18414995074272156, 0.18712753057479858, 0.18857455253601074], "cos_Ac_negg": [0.47644031047821045, 0.4332488179206848, 0.3458453416824341], "r_norm": [0.7485713958740234, 0.735547661781311, 0.7477411031723022]}, {"step": 300, "epoch": 0, "train_loss": 0.14099109172821045, "cos_r_negg": [0.4927925169467926, 0.46113651990890503, 0.37848058342933655], "cos_innovation_negg": [0.4927925169467926, 0.46113651990890503, 0.37848058342933655], "cos_apical_negg": [0.16623306274414062, 0.17412245273590088, 0.18429788947105408], "cos_Ac_negg": [0.4929081201553345, 0.461198091506958, 0.37851786613464355], "r_norm": [0.6309524178504944, 0.6242880821228027, 0.6307910680770874]}, {"step": 400, "epoch": 0, "train_loss": 0.17211204767227173, "cos_r_negg": [0.5215590596199036, 0.47604459524154663, 0.41393402218818665], "cos_innovation_negg": [0.5215590596199036, 0.47604459524154663, 0.41393402218818665], "cos_apical_negg": [0.15252500772476196, 0.16053888201713562, 0.18281841278076172], "cos_Ac_negg": [0.5218977928161621, 0.4759637117385864, 0.41391241550445557], "r_norm": [0.5391026735305786, 0.5325049161911011, 0.5389631986618042]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9454, "test_loss": 0.18038152313232422}, {"step": 500, "epoch": 1, "train_loss": 0.22109106183052063, "cos_r_negg": [0.5123040676116943, 0.499828040599823, 0.42953142523765564], "cos_innovation_negg": [0.5123040676116943, 0.499828040599823, 0.42953142523765564], "cos_apical_negg": [0.13247118890285492, 0.1363692432641983, 0.1852826178073883], "cos_Ac_negg": [0.5127270221710205, 0.49992114305496216, 0.4294663071632385], "r_norm": [0.5086184144020081, 0.49894434213638306, 0.5069912672042847], "ldr": {"dL_sdil": -0.014748841524124146, "dL_bp": -0.012521758675575256, "ratio": 1.177857033205168}}, {"step": 600, "epoch": 1, "train_loss": 0.10845459252595901, "cos_r_negg": [0.5136754512786865, 0.5219959020614624, 0.4420141577720642], "cos_innovation_negg": [0.5136754512786865, 0.5219959020614624, 0.4420141577720642], "cos_apical_negg": [0.11937592923641205, 0.13484901189804077, 0.17201198637485504], "cos_Ac_negg": [0.5142444372177124, 0.5224712491035461, 0.4419541358947754], "r_norm": [0.4019564986228943, 0.3954637050628662, 0.4005798101425171]}, {"step": 700, "epoch": 1, "train_loss": 0.20623335242271423, "cos_r_negg": [0.544032871723175, 0.5368605852127075, 0.44347792863845825], "cos_innovation_negg": [0.544032871723175, 0.5368605852127075, 0.44347792863845825], "cos_apical_negg": [0.1390547901391983, 0.15632595121860504, 0.18349575996398926], "cos_Ac_negg": [0.5453894138336182, 0.5376474857330322, 0.44339117407798767], "r_norm": [0.4327465295791626, 0.42756372690200806, 0.4326320290565491]}, {"step": 800, "epoch": 1, "train_loss": 0.1443621665239334, "cos_r_negg": [0.5252273082733154, 0.5320252180099487, 0.47417280077934265], "cos_innovation_negg": [0.5252273082733154, 0.5320252180099487, 0.47417280077934265], "cos_apical_negg": [0.11338149756193161, 0.12324408441781998, 0.172146275639534], "cos_Ac_negg": [0.5257970094680786, 0.5325189232826233, 0.4741213619709015], "r_norm": [0.35672813653945923, 0.3530079126358032, 0.35578417778015137]}, {"step": 900, "epoch": 1, "train_loss": 0.21021562814712524, "cos_r_negg": [0.5338379144668579, 0.5530998110771179, 0.4773702919483185], "cos_innovation_negg": [0.5338379144668579, 0.5530998110771179, 0.4773702919483185], "cos_apical_negg": [0.130013108253479, 0.14560917019844055, 0.16989204287528992], "cos_Ac_negg": [0.533989429473877, 0.5531902313232422, 0.47731107473373413], "r_norm": [0.41594284772872925, 0.4096883535385132, 0.4138052463531494]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9563, "test_loss": 0.1548952980041504}, {"step": 1000, "epoch": 2, "train_loss": 0.013534436002373695, "cos_r_negg": [0.5393010973930359, 0.5528655648231506, 0.4911356568336487], "cos_innovation_negg": [0.5393010973930359, 0.5528655648231506, 0.4911356568336487], "cos_apical_negg": [0.09154845774173737, 0.11205717921257019, 0.1610165536403656], "cos_Ac_negg": [0.5395617485046387, 0.553184986114502, 0.4907969832420349], "r_norm": [0.31779003143310547, 0.31518927216529846, 0.3174085021018982], "ldr": {"dL_sdil": -0.0063033923506736755, "dL_bp": -0.005692437291145325, "ratio": 1.1073274993259392}}, {"step": 1100, "epoch": 2, "train_loss": 0.14892613887786865, "cos_r_negg": [0.5432581901550293, 0.5739394426345825, 0.5088933110237122], "cos_innovation_negg": [0.5432581901550293, 0.5739394426345825, 0.5088933110237122], "cos_apical_negg": [0.10961363464593887, 0.13377754390239716, 0.17714881896972656], "cos_Ac_negg": [0.5437158942222595, 0.5740728378295898, 0.5084002614021301], "r_norm": [0.3903537690639496, 0.3856971859931946, 0.38598519563674927]}, {"step": 1200, "epoch": 2, "train_loss": 0.1322353631258011, "cos_r_negg": [0.5249543190002441, 0.5608853697776794, 0.5076075196266174], "cos_innovation_negg": [0.5249543190002441, 0.5608853697776794, 0.5076075196266174], "cos_apical_negg": [0.07521849870681763, 0.11062383651733398, 0.15268629789352417], "cos_Ac_negg": [0.5263264179229736, 0.5619728565216064, 0.5070093870162964], "r_norm": [0.33100879192352295, 0.3271823823451996, 0.3293168544769287]}, {"step": 1300, "epoch": 2, "train_loss": 0.1248982772231102, "cos_r_negg": [0.5413243174552917, 0.5811703205108643, 0.512839138507843], "cos_innovation_negg": [0.5413243174552917, 0.5811703205108643, 0.512839138507843], "cos_apical_negg": [0.10059108585119247, 0.12123358249664307, 0.15553097426891327], "cos_Ac_negg": [0.5420103073120117, 0.581811785697937, 0.5120610594749451], "r_norm": [0.31863194704055786, 0.3162306547164917, 0.31757181882858276]}, {"step": 1400, "epoch": 2, "train_loss": 0.15081247687339783, "cos_r_negg": [0.5458354949951172, 0.5704351663589478, 0.5030908584594727], "cos_innovation_negg": [0.5458354949951172, 0.5704351663589478, 0.5030908584594727], "cos_apical_negg": [0.0876942053437233, 0.11402508616447449, 0.1553117036819458], "cos_Ac_negg": [0.5477430820465088, 0.5731785297393799, 0.5030618906021118], "r_norm": [0.29823172092437744, 0.29524749517440796, 0.2968723177909851]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9645, "test_loss": 0.12388780097961426}, {"step": 1500, "epoch": 3, "train_loss": 0.04028237611055374, "cos_r_negg": [0.5587222576141357, 0.5883270502090454, 0.5224758386611938], "cos_innovation_negg": [0.5587222576141357, 0.5883270502090454, 0.5224758386611938], "cos_apical_negg": [0.06787993013858795, 0.1011337861418724, 0.15038762986660004], "cos_Ac_negg": [0.5597240924835205, 0.5889119505882263, 0.5218582153320312], "r_norm": [0.2850165367126465, 0.2799333930015564, 0.2821390628814697], "ldr": {"dL_sdil": -0.006664574146270752, "dL_bp": -0.006693407893180847, "ratio": 0.9956922172725391}}, {"step": 1600, "epoch": 3, "train_loss": 0.10059386491775513, "cos_r_negg": [0.5654084086418152, 0.5908755660057068, 0.5266769528388977], "cos_innovation_negg": [0.5654084086418152, 0.5908755660057068, 0.5266769528388977], "cos_apical_negg": [0.08414043486118317, 0.10758581757545471, 0.14044392108917236], "cos_Ac_negg": [0.5662648677825928, 0.5915886759757996, 0.5257163643836975], "r_norm": [0.2612973749637604, 0.25870954990386963, 0.25837814807891846]}, {"step": 1700, "epoch": 3, "train_loss": 0.040523283183574677, "cos_r_negg": [0.5497581958770752, 0.5974898338317871, 0.5349857807159424], "cos_innovation_negg": [0.5497581958770752, 0.5974898338317871, 0.5349857807159424], "cos_apical_negg": [0.08124493062496185, 0.11289441585540771, 0.1483648717403412], "cos_Ac_negg": [0.5545519590377808, 0.6012608408927917, 0.5365568399429321], "r_norm": [0.2678019404411316, 0.26343464851379395, 0.2634611129760742]}, {"step": 1800, "epoch": 3, "train_loss": 0.027563145384192467, "cos_r_negg": [0.5687366724014282, 0.6049169898033142, 0.5381788611412048], "cos_innovation_negg": [0.5687366724014282, 0.6049169898033142, 0.5381788611412048], "cos_apical_negg": [0.049595482647418976, 0.09417969733476639, 0.14538456499576569], "cos_Ac_negg": [0.5730600953102112, 0.6076704263687134, 0.5386600494384766], "r_norm": [0.21144218742847443, 0.20892775058746338, 0.20958486199378967]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9635, "test_loss": 0.12646708030700685}, {"step": 1900, "epoch": 4, "train_loss": 0.015225281938910484, "cos_r_negg": [0.5637871026992798, 0.6106384992599487, 0.5436741709709167], "cos_innovation_negg": [0.5637871026992798, 0.6106384992599487, 0.5436741709709167], "cos_apical_negg": [0.07135987281799316, 0.12155803292989731, 0.15595504641532898], "cos_Ac_negg": [0.5693656206130981, 0.6151727437973022, 0.5434404611587524], "r_norm": [0.3249160945415497, 0.32185566425323486, 0.3233655095100403]}, {"step": 2000, "epoch": 4, "train_loss": 0.11175475269556046, "cos_r_negg": [0.5507205724716187, 0.5885815620422363, 0.5400558114051819], "cos_innovation_negg": [0.5507205724716187, 0.5885815620422363, 0.5400558114051819], "cos_apical_negg": [0.054052963852882385, 0.08648832887411118, 0.136165052652359], "cos_Ac_negg": [0.5646088123321533, 0.599081814289093, 0.544467568397522], "r_norm": [0.23077525198459625, 0.22606392204761505, 0.2264135181903839], "ldr": {"dL_sdil": -0.00494229793548584, "dL_bp": -0.004889249801635742, "ratio": 1.0108499536743552}}, {"step": 2100, "epoch": 4, "train_loss": 0.018824519589543343, "cos_r_negg": [0.5495516061782837, 0.6029183864593506, 0.5498232245445251], "cos_innovation_negg": [0.5495516061782837, 0.6029183864593506, 0.5498232245445251], "cos_apical_negg": [0.05941082537174225, 0.1011587381362915, 0.13563662767410278], "cos_Ac_negg": [0.5616047978401184, 0.6121972799301147, 0.5551217794418335], "r_norm": [0.2296755611896515, 0.22587710618972778, 0.22697503864765167]}, {"step": 2200, "epoch": 4, "train_loss": 0.020374715328216553, "cos_r_negg": [0.5610834360122681, 0.6179388165473938, 0.5692180395126343], "cos_innovation_negg": [0.5610834360122681, 0.6179388165473938, 0.5692180395126343], "cos_apical_negg": [0.04980545490980148, 0.09287083148956299, 0.14041882753372192], "cos_Ac_negg": [0.5652894973754883, 0.6210495233535767, 0.5688716173171997], "r_norm": [0.23919302225112915, 0.23531633615493774, 0.23693540692329407]}, {"step": 2300, "epoch": 4, "train_loss": 0.1461770385503769, "cos_r_negg": [0.5443936586380005, 0.6041824817657471, 0.5717413425445557], "cos_innovation_negg": [0.5443936586380005, 0.6041824817657471, 0.5717413425445557], "cos_apical_negg": [0.06246369332075119, 0.08481846749782562, 0.13933628797531128], "cos_Ac_negg": [0.5540826320648193, 0.6111220121383667, 0.5723639726638794], "r_norm": [0.23322424292564392, 0.23028363287448883, 0.23023328185081482]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9693, "test_loss": 0.11612348155975342}, {"step": 2400, "epoch": 5, "train_loss": 0.04837659373879433, "cos_r_negg": [0.5497555136680603, 0.613798975944519, 0.5836246013641357], "cos_innovation_negg": [0.5497555136680603, 0.613798975944519, 0.5836246013641357], "cos_apical_negg": [0.03422921150922775, 0.07524777203798294, 0.1288151741027832], "cos_Ac_negg": [0.5625805854797363, 0.6243216395378113, 0.5868514776229858], "r_norm": [0.1937330812215805, 0.1903836578130722, 0.1914653182029724]}, {"step": 2500, "epoch": 5, "train_loss": 0.03572862967848778, "cos_r_negg": [0.536297619342804, 0.6122587323188782, 0.5744787454605103], "cos_innovation_negg": [0.536297619342804, 0.6122587323188782, 0.5744787454605103], "cos_apical_negg": [0.04073914885520935, 0.08260023593902588, 0.13423827290534973], "cos_Ac_negg": [0.5574368238449097, 0.6312673687934875, 0.5834949016571045], "r_norm": [0.22491031885147095, 0.22121581435203552, 0.22201752662658691], "ldr": {"dL_sdil": -0.006825193762779236, "dL_bp": -0.008200183510780334, "ratio": 0.8323220759397062}}, {"step": 2600, "epoch": 5, "train_loss": 0.06313536316156387, "cos_r_negg": [0.5575818419456482, 0.6159696578979492, 0.5860693454742432], "cos_innovation_negg": [0.5575818419456482, 0.6159696578979492, 0.5860693454742432], "cos_apical_negg": [0.044788509607315063, 0.07898969948291779, 0.13901212811470032], "cos_Ac_negg": [0.5775165557861328, 0.6329346895217896, 0.5903804302215576], "r_norm": [0.2355024516582489, 0.23218277096748352, 0.23283711075782776]}, {"step": 2700, "epoch": 5, "train_loss": 0.06016786769032478, "cos_r_negg": [0.5621662139892578, 0.6278358697891235, 0.5985808372497559], "cos_innovation_negg": [0.5621662139892578, 0.6278358697891235, 0.5985808372497559], "cos_apical_negg": [0.035947442054748535, 0.08550694584846497, 0.13080622255802155], "cos_Ac_negg": [0.5798708200454712, 0.6431190371513367, 0.6033709049224854], "r_norm": [0.18281646072864532, 0.17833128571510315, 0.17989735305309296]}, {"step": 2800, "epoch": 5, "train_loss": 0.06002722680568695, "cos_r_negg": [0.5565155744552612, 0.6199231147766113, 0.5964016318321228], "cos_innovation_negg": [0.5565155744552612, 0.6199231147766113, 0.5964016318321228], "cos_apical_negg": [0.056141890585422516, 0.07961973547935486, 0.13654595613479614], "cos_Ac_negg": [0.5821346640586853, 0.6405961513519287, 0.6023705005645752], "r_norm": [0.21004138886928558, 0.2063857614994049, 0.20794540643692017]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9691, "test_loss": 0.117822243309021}, {"step": 2900, "epoch": 6, "train_loss": 0.004632744938135147, "cos_r_negg": [0.5625027418136597, 0.6185270547866821, 0.5973436832427979], "cos_innovation_negg": [0.5625027418136597, 0.6185270547866821, 0.5973436832427979], "cos_apical_negg": [0.04439803957939148, 0.07868873327970505, 0.1325622797012329], "cos_Ac_negg": [0.5979520678520203, 0.6471431255340576, 0.6128271818161011], "r_norm": [0.20341899991035461, 0.19988590478897095, 0.2015589028596878]}, {"step": 3000, "epoch": 6, "train_loss": 0.10176092386245728, "cos_r_negg": [0.5381157994270325, 0.596107006072998, 0.592484712600708], "cos_innovation_negg": [0.5381157994270325, 0.596107006072998, 0.592484712600708], "cos_apical_negg": [0.04250746965408325, 0.06732872128486633, 0.12799181044101715], "cos_Ac_negg": [0.5829313397407532, 0.6410611867904663, 0.6182559728622437], "r_norm": [0.18036162853240967, 0.17583096027374268, 0.17725782096385956], "ldr": {"dL_sdil": -0.0044542476534843445, "dL_bp": -0.005535684525966644, "ratio": 0.8046426115127182}}, {"step": 3100, "epoch": 6, "train_loss": 0.046246424317359924, "cos_r_negg": [0.5313170552253723, 0.5852774381637573, 0.5927698612213135], "cos_innovation_negg": [0.5313170552253723, 0.5852774381637573, 0.5927698612213135], "cos_apical_negg": [0.04007897526025772, 0.06363086402416229, 0.12532269954681396], "cos_Ac_negg": [0.5821939706802368, 0.640598475933075, 0.6236097812652588], "r_norm": [0.16094055771827698, 0.15582650899887085, 0.15900975465774536]}, {"step": 3200, "epoch": 6, "train_loss": 0.016133006662130356, "cos_r_negg": [0.5231339335441589, 0.5813295841217041, 0.5877962112426758], "cos_innovation_negg": [0.5231339335441589, 0.5813295841217041, 0.5877962112426758], "cos_apical_negg": [0.03526952490210533, 0.06485208868980408, 0.1240764632821083], "cos_Ac_negg": [0.5881475210189819, 0.6474661827087402, 0.6253440380096436], "r_norm": [0.17031420767307281, 0.16718390583992004, 0.1690060943365097]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9707, "test_loss": 0.115691064453125}, {"step": 3300, "epoch": 7, "train_loss": 0.0074622053653001785, "cos_r_negg": [0.5143393874168396, 0.5768665671348572, 0.582038164138794], "cos_innovation_negg": [0.5143393874168396, 0.5768665671348572, 0.582038164138794], "cos_apical_negg": [0.03136651962995529, 0.061983153223991394, 0.11461574584245682], "cos_Ac_negg": [0.5870507955551147, 0.6505273580551147, 0.6293970942497253], "r_norm": [0.1587715744972229, 0.15534371137619019, 0.15824705362319946]}, {"step": 3400, "epoch": 7, "train_loss": 0.01476661954075098, "cos_r_negg": [0.48588258028030396, 0.5407200455665588, 0.5630463361740112], "cos_innovation_negg": [0.48588258028030396, 0.5407200455665588, 0.5630463361740112], "cos_apical_negg": [0.0291840061545372, 0.06487597525119781, 0.12235292792320251], "cos_Ac_negg": [0.5872980952262878, 0.6487741470336914, 0.6343486309051514], "r_norm": [0.17334650456905365, 0.1690506488084793, 0.17054972052574158]}, {"step": 3500, "epoch": 7, "train_loss": 0.006560184061527252, "cos_r_negg": [0.48604172468185425, 0.5459116697311401, 0.5594459772109985], "cos_innovation_negg": [0.48604172468185425, 0.5459116697311401, 0.5594459772109985], "cos_apical_negg": [0.03325795382261276, 0.05744883790612221, 0.11568857729434967], "cos_Ac_negg": [0.5845441818237305, 0.6544637084007263, 0.6349652409553528], "r_norm": [0.1733866035938263, 0.1700642853975296, 0.17158269882202148], "ldr": {"dL_sdil": -0.006456710398197174, "dL_bp": -0.009141050279140472, "ratio": 0.7063422912059832}}, {"step": 3600, "epoch": 7, "train_loss": 0.001369433244690299, "cos_r_negg": [0.48910605907440186, 0.5545433163642883, 0.5670478940010071], "cos_innovation_negg": [0.48910605907440186, 0.5545433163642883, 0.5670478940010071], "cos_apical_negg": [0.03457564860582352, 0.06288737058639526, 0.11744757741689682], "cos_Ac_negg": [0.5871787667274475, 0.6595185399055481, 0.6443623900413513], "r_norm": [0.17093980312347412, 0.1654106080532074, 0.16844306886196136]}, {"step": 3700, "epoch": 7, "train_loss": 0.0021250678692013025, "cos_r_negg": [0.4907078146934509, 0.5503069162368774, 0.5675644874572754], "cos_innovation_negg": [0.4907078146934509, 0.5503069162368774, 0.5675644874572754], "cos_apical_negg": [0.052500028163194656, 0.06807701289653778, 0.1133408397436142], "cos_Ac_negg": [0.5924239158630371, 0.6609795689582825, 0.6507213711738586], "r_norm": [0.18370124697685242, 0.1790221631526947, 0.18143978714942932]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9726, "test_loss": 0.12102063503265381}, {"step": 3800, "epoch": 8, "train_loss": 0.002206292701885104, "cos_r_negg": [0.455464243888855, 0.5244035124778748, 0.548377275466919], "cos_innovation_negg": [0.455464243888855, 0.5244035124778748, 0.548377275466919], "cos_apical_negg": [0.033319875597953796, 0.05864904075860977, 0.11244694888591766], "cos_Ac_negg": [0.5827034711837769, 0.6586761474609375, 0.6472840309143066], "r_norm": [0.16320034861564636, 0.16059494018554688, 0.16055868566036224]}, {"step": 3900, "epoch": 8, "train_loss": 0.0057874200865626335, "cos_r_negg": [0.4457911252975464, 0.5177236199378967, 0.5387643575668335], "cos_innovation_negg": [0.4457911252975464, 0.5177236199378967, 0.5387643575668335], "cos_apical_negg": [0.030217789113521576, 0.06493405997753143, 0.11017754673957825], "cos_Ac_negg": [0.5799587965011597, 0.6631183624267578, 0.6538204550743103], "r_norm": [0.14894600212574005, 0.14518943428993225, 0.14689865708351135]}, {"step": 4000, "epoch": 8, "train_loss": 0.006218268536031246, "cos_r_negg": [0.4253624379634857, 0.4945695996284485, 0.5257371664047241], "cos_innovation_negg": [0.4253624379634857, 0.4945695996284485, 0.5257371664047241], "cos_apical_negg": [0.0323532298207283, 0.058745063841342926, 0.10401240736246109], "cos_Ac_negg": [0.5789551138877869, 0.6598927974700928, 0.6540126204490662], "r_norm": [0.15835723280906677, 0.15524335205554962, 0.1564338654279709], "ldr": {"dL_sdil": -0.006487831473350525, "dL_bp": -0.009858623147010803, "ratio": 0.6580869738709585}}, {"step": 4100, "epoch": 8, "train_loss": 0.01178626250475645, "cos_r_negg": [0.43771135807037354, 0.508246660232544, 0.5339679718017578], "cos_innovation_negg": [0.43771135807037354, 0.508246660232544, 0.5339679718017578], "cos_apical_negg": [0.03737693279981613, 0.06301218271255493, 0.10186943411827087], "cos_Ac_negg": [0.5784783363342285, 0.660437822341919, 0.6590499877929688], "r_norm": [0.14424073696136475, 0.14067167043685913, 0.14295223355293274]}, {"step": 4200, "epoch": 8, "train_loss": 0.014768831431865692, "cos_r_negg": [0.4162524342536926, 0.4912894070148468, 0.5207354426383972], "cos_innovation_negg": [0.4162524342536926, 0.4912894070148468, 0.5207354426383972], "cos_apical_negg": [0.02151874080300331, 0.05593392625451088, 0.0995345264673233], "cos_Ac_negg": [0.5789660811424255, 0.6606719493865967, 0.6565781831741333], "r_norm": [0.15473459661006927, 0.15114256739616394, 0.1521206647157669]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9725, "test_loss": 0.12661395950317383}, {"step": 4300, "epoch": 9, "train_loss": 0.0013256785459816456, "cos_r_negg": [0.40297722816467285, 0.46929240226745605, 0.5026384592056274], "cos_innovation_negg": [0.40297722816467285, 0.46929240226745605, 0.5026384592056274], "cos_apical_negg": [0.02145608514547348, 0.05844983458518982, 0.09758409857749939], "cos_Ac_negg": [0.5759316682815552, 0.6565595865249634, 0.6569873094558716], "r_norm": [0.15959107875823975, 0.15707780420780182, 0.15762834250926971]}, {"step": 4400, "epoch": 9, "train_loss": 0.0028267870657145977, "cos_r_negg": [0.384077250957489, 0.45126840472221375, 0.49205225706100464], "cos_innovation_negg": [0.384077250957489, 0.45126840472221375, 0.49205225706100464], "cos_apical_negg": [0.019026050344109535, 0.05792634189128876, 0.09886044263839722], "cos_Ac_negg": [0.5747684836387634, 0.656345009803772, 0.6619800329208374], "r_norm": [0.17581918835639954, 0.17166317999362946, 0.17315301299095154]}, {"step": 4500, "epoch": 9, "train_loss": 0.05075601115822792, "cos_r_negg": [0.3887633979320526, 0.4507681727409363, 0.4911920130252838], "cos_innovation_negg": [0.3887633979320526, 0.4507681727409363, 0.4911920130252838], "cos_apical_negg": [0.026776937767863274, 0.05881907790899277, 0.10061895102262497], "cos_Ac_negg": [0.5810046195983887, 0.6585254669189453, 0.6648799180984497], "r_norm": [0.15796691179275513, 0.15528523921966553, 0.15562662482261658], "ldr": {"dL_sdil": -0.007897064089775085, "dL_bp": -0.012957476079463959, "ratio": 0.609460055441737}}, {"step": 4600, "epoch": 9, "train_loss": 0.0005691738333553076, "cos_r_negg": [0.38581597805023193, 0.45048877596855164, 0.4899650812149048], "cos_innovation_negg": [0.38581597805023193, 0.45048877596855164, 0.4899650812149048], "cos_apical_negg": [0.023280508816242218, 0.05489152669906616, 0.09833288192749023], "cos_Ac_negg": [0.5753306150436401, 0.6580355167388916, 0.6644841432571411], "r_norm": [0.15791055560112, 0.1558215171098709, 0.15515026450157166]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9731, "test_loss": 0.13092050895690918}, {"step": 4700, "epoch": 10, "train_loss": 0.0037356780376285315, "cos_r_negg": [0.36362963914871216, 0.4224255084991455, 0.4594394266605377], "cos_innovation_negg": [0.36362963914871216, 0.4224255084991455, 0.4594394266605377], "cos_apical_negg": [0.02317235991358757, 0.055542632937431335, 0.09649364650249481], "cos_Ac_negg": [0.5738381743431091, 0.6560713648796082, 0.6588197946548462], "r_norm": [0.16464504599571228, 0.16251137852668762, 0.1628788262605667]}, {"step": 4800, "epoch": 10, "train_loss": 0.0034398639108985662, "cos_r_negg": [0.3516177535057068, 0.4107159972190857, 0.444058358669281], "cos_innovation_negg": [0.3516177535057068, 0.4107159972190857, 0.444058358669281], "cos_apical_negg": [0.018939707428216934, 0.056151412427425385, 0.09089028835296631], "cos_Ac_negg": [0.5741580128669739, 0.6540528535842896, 0.6597457528114319], "r_norm": [0.15535713732242584, 0.1531437784433365, 0.15327326953411102]}, {"step": 4900, "epoch": 10, "train_loss": 0.001210769871249795, "cos_r_negg": [0.3454546332359314, 0.40243300795555115, 0.4382479190826416], "cos_innovation_negg": [0.3454546332359314, 0.40243300795555115, 0.4382479190826416], "cos_apical_negg": [0.013848450034856796, 0.05157997086644173, 0.0892142504453659], "cos_Ac_negg": [0.5730417966842651, 0.6554983854293823, 0.6587049961090088], "r_norm": [0.13984805345535278, 0.13772936165332794, 0.13778550922870636]}, {"step": 5000, "epoch": 10, "train_loss": 0.00034295825753360987, "cos_r_negg": [0.3217296600341797, 0.3829406499862671, 0.4185682535171509], "cos_innovation_negg": [0.3217296600341797, 0.3829406499862671, 0.4185682535171509], "cos_apical_negg": [0.01483751367777586, 0.05268975719809532, 0.08780761808156967], "cos_Ac_negg": [0.5671790242195129, 0.6507988572120667, 0.657765805721283], "r_norm": [0.1580510288476944, 0.1568574756383896, 0.15615391731262207], "ldr": {"dL_sdil": -0.008800826966762543, "dL_bp": -0.01480904221534729, "ratio": 0.5942873846116693}}, {"step": 5100, "epoch": 10, "train_loss": 0.00383337982930243, "cos_r_negg": [0.34335386753082275, 0.40398091077804565, 0.44169560074806213], "cos_innovation_negg": [0.34335386753082275, 0.40398091077804565, 0.44169560074806213], "cos_apical_negg": [0.014041442424058914, 0.05189131200313568, 0.08587805181741714], "cos_Ac_negg": [0.5753651261329651, 0.655285120010376, 0.6660525798797607], "r_norm": [0.14091816544532776, 0.13953402638435364, 0.13897103071212769]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9721, "test_loss": 0.1377161819458008}, {"step": 5200, "epoch": 11, "train_loss": 0.0006252616876736283, "cos_r_negg": [0.3330591320991516, 0.39026546478271484, 0.430513471364975], "cos_innovation_negg": [0.3330591320991516, 0.39026546478271484, 0.430513471364975], "cos_apical_negg": [0.01826108805835247, 0.04962429404258728, 0.09033550322055817], "cos_Ac_negg": [0.5750359892845154, 0.6553247570991516, 0.6685219407081604], "r_norm": [0.15488600730895996, 0.1532621681690216, 0.1529770940542221]}, {"step": 5300, "epoch": 11, "train_loss": 0.001262085628695786, "cos_r_negg": [0.3209419846534729, 0.37671148777008057, 0.41460204124450684], "cos_innovation_negg": [0.3209419846534729, 0.37671148777008057, 0.41460204124450684], "cos_apical_negg": [0.015010406263172626, 0.04841785505414009, 0.08628202974796295], "cos_Ac_negg": [0.5682228207588196, 0.6508738398551941, 0.6643944978713989], "r_norm": [0.1570015549659729, 0.15463638305664062, 0.15491819381713867]}, {"step": 5400, "epoch": 11, "train_loss": 0.0020298766903579235, "cos_r_negg": [0.31819993257522583, 0.3732580244541168, 0.40698209404945374], "cos_innovation_negg": [0.31819993257522583, 0.3732580244541168, 0.40698209404945374], "cos_apical_negg": [0.01870063878595829, 0.05015512555837631, 0.08647607266902924], "cos_Ac_negg": [0.5702471137046814, 0.6530635952949524, 0.6635887622833252], "r_norm": [0.1563768982887268, 0.1544710099697113, 0.15432460606098175]}, {"step": 5500, "epoch": 11, "train_loss": 0.0023977679666131735, "cos_r_negg": [0.3127661347389221, 0.3656383156776428, 0.40201860666275024], "cos_innovation_negg": [0.3127661347389221, 0.3656383156776428, 0.40201860666275024], "cos_apical_negg": [0.014782181940972805, 0.04763004928827286, 0.0845022201538086], "cos_Ac_negg": [0.5699431896209717, 0.650721549987793, 0.6632071733474731], "r_norm": [0.15096385776996613, 0.14922848343849182, 0.14893195033073425], "ldr": {"dL_sdil": -0.00817299634218216, "dL_bp": -0.014765284955501556, "ratio": 0.5535278436422518}}, {"step": 5600, "epoch": 11, "train_loss": 0.0005157387931831181, "cos_r_negg": [0.3039851784706116, 0.3539344072341919, 0.38751694560050964], "cos_innovation_negg": [0.3039851784706116, 0.3539344072341919, 0.38751694560050964], "cos_apical_negg": [0.01567096635699272, 0.04664938896894455, 0.08173708617687225], "cos_Ac_negg": [0.5669693946838379, 0.6515830755233765, 0.661815881729126], "r_norm": [0.15070994198322296, 0.14903470873832703, 0.1486593782901764]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9727, "test_loss": 0.1386508232116699}, {"step": 5700, "epoch": 12, "train_loss": 0.0005900029791519046, "cos_r_negg": [0.30520695447921753, 0.3533356785774231, 0.3877496123313904], "cos_innovation_negg": [0.30520695447921753, 0.3533356785774231, 0.3877496123313904], "cos_apical_negg": [0.017848851159214973, 0.04629286751151085, 0.08183339238166809], "cos_Ac_negg": [0.567740261554718, 0.6493000984191895, 0.6629624366760254], "r_norm": [0.1527186930179596, 0.15123730897903442, 0.1508171707391739]}, {"step": 5800, "epoch": 12, "train_loss": 0.00045360904186964035, "cos_r_negg": [0.29937607049942017, 0.34640997648239136, 0.3820086419582367], "cos_innovation_negg": [0.29937607049942017, 0.34640997648239136, 0.3820086419582367], "cos_apical_negg": [0.01712479442358017, 0.04618170112371445, 0.08015058189630508], "cos_Ac_negg": [0.5681700706481934, 0.6509140133857727, 0.6639939546585083], "r_norm": [0.15499885380268097, 0.1535048633813858, 0.15306100249290466]}, {"step": 5900, "epoch": 12, "train_loss": 0.001925840275362134, "cos_r_negg": [0.2941708564758301, 0.3443789780139923, 0.37495964765548706], "cos_innovation_negg": [0.2941708564758301, 0.3443789780139923, 0.37495964765548706], "cos_apical_negg": [0.012477493844926357, 0.04545557126402855, 0.07571226358413696], "cos_Ac_negg": [0.5679708123207092, 0.6500318050384521, 0.6636041402816772], "r_norm": [0.14870283007621765, 0.14706546068191528, 0.14679986238479614]}, {"step": 6000, "epoch": 12, "train_loss": 0.000872130214702338, "cos_r_negg": [0.28958576917648315, 0.33663880825042725, 0.36483848094940186], "cos_innovation_negg": [0.28958576917648315, 0.33663880825042725, 0.36483848094940186], "cos_apical_negg": [0.01279729325324297, 0.0444515198469162, 0.07291647791862488], "cos_Ac_negg": [0.5679613947868347, 0.6489011645317078, 0.6622161865234375], "r_norm": [0.15145450830459595, 0.15000420808792114, 0.14951230585575104], "ldr": {"dL_sdil": -0.008593045175075531, "dL_bp": -0.016068294644355774, "ratio": 0.5347826490158347}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9729, "test_loss": 0.1413892677307129}, {"step": 6100, "epoch": 13, "train_loss": 0.0005852764006704092, "cos_r_negg": [0.28444358706474304, 0.3295268714427948, 0.3606950640678406], "cos_innovation_negg": [0.28444358706474304, 0.3295268714427948, 0.3606950640678406], "cos_apical_negg": [0.0094082560390234, 0.04211698845028877, 0.07285453379154205], "cos_Ac_negg": [0.5689363479614258, 0.649165153503418, 0.6642417907714844], "r_norm": [0.1477573812007904, 0.1461597979068756, 0.14574216306209564]}, {"step": 6200, "epoch": 13, "train_loss": 0.0008907088777050376, "cos_r_negg": [0.2747124433517456, 0.3212924003601074, 0.3519812226295471], "cos_innovation_negg": [0.2747124433517456, 0.3212924003601074, 0.3519812226295471], "cos_apical_negg": [0.01017395593225956, 0.04224824160337448, 0.07193300127983093], "cos_Ac_negg": [0.567864179611206, 0.6480947136878967, 0.663794994354248], "r_norm": [0.1496768742799759, 0.14803925156593323, 0.14763718843460083]}, {"step": 6300, "epoch": 13, "train_loss": 0.0002600089064799249, "cos_r_negg": [0.2706194519996643, 0.3153204023838043, 0.3434497117996216], "cos_innovation_negg": [0.2706194519996643, 0.3153204023838043, 0.3434497117996216], "cos_apical_negg": [0.010187894105911255, 0.040824972093105316, 0.0685247927904129], "cos_Ac_negg": [0.5668900012969971, 0.6464422941207886, 0.6632004380226135], "r_norm": [0.1528550088405609, 0.15130236744880676, 0.1507633477449417]}, {"step": 6400, "epoch": 13, "train_loss": 0.0005894043715670705, "cos_r_negg": [0.2727878987789154, 0.3177923560142517, 0.3482232689857483], "cos_innovation_negg": [0.2727878987789154, 0.3177923560142517, 0.3482232689857483], "cos_apical_negg": [0.0107734315097332, 0.044099241495132446, 0.07181450724601746], "cos_Ac_negg": [0.56566321849823, 0.6468288898468018, 0.6630930304527283], "r_norm": [0.1559704691171646, 0.15443193912506104, 0.15403473377227783]}, {"step": 6500, "epoch": 13, "train_loss": 0.00013936757750343531, "cos_r_negg": [0.27071112394332886, 0.31402087211608887, 0.3444286286830902], "cos_innovation_negg": [0.27071112394332886, 0.31402087211608887, 0.3444286286830902], "cos_apical_negg": [0.012675473466515541, 0.04204685240983963, 0.06844287365674973], "cos_Ac_negg": [0.5677371025085449, 0.6475487947463989, 0.6652024984359741], "r_norm": [0.15206179022789001, 0.1506393849849701, 0.15023097395896912], "ldr": {"dL_sdil": -0.00946023315191269, "dL_bp": -0.01796794682741165, "ratio": 0.5265060745549562}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9741, "test_loss": 0.14270558700561523}, {"step": 6600, "epoch": 14, "train_loss": 0.0002548241172917187, "cos_r_negg": [0.2635860741138458, 0.30646800994873047, 0.33641529083251953], "cos_innovation_negg": [0.2635860741138458, 0.30646800994873047, 0.33641529083251953], "cos_apical_negg": [0.0109136663377285, 0.04128178209066391, 0.0669466033577919], "cos_Ac_negg": [0.5665901899337769, 0.6470203399658203, 0.6646734476089478], "r_norm": [0.15558183193206787, 0.15414045751094818, 0.1536445915699005]}, {"step": 6700, "epoch": 14, "train_loss": 0.0007273289375007153, "cos_r_negg": [0.2616570293903351, 0.30892568826675415, 0.33903396129608154], "cos_innovation_negg": [0.2616570293903351, 0.30892568826675415, 0.33903396129608154], "cos_apical_negg": [0.008511800318956375, 0.04415461793541908, 0.06810304522514343], "cos_Ac_negg": [0.565061628818512, 0.6471189260482788, 0.6661461591720581], "r_norm": [0.1566920429468155, 0.15524893999099731, 0.15486150979995728]}, {"step": 6800, "epoch": 14, "train_loss": 0.0005857844953425229, "cos_r_negg": [0.25264739990234375, 0.29708677530288696, 0.32618093490600586], "cos_innovation_negg": [0.25264739990234375, 0.29708677530288696, 0.32618093490600586], "cos_apical_negg": [0.008597392588853836, 0.04278518632054329, 0.06489677727222443], "cos_Ac_negg": [0.564162015914917, 0.6456236839294434, 0.6628995537757874], "r_norm": [0.15494146943092346, 0.15355153381824493, 0.15291886031627655]}, {"step": 6900, "epoch": 14, "train_loss": 0.0008804219542071223, "cos_r_negg": [0.2492126226425171, 0.29383403062820435, 0.32258570194244385], "cos_innovation_negg": [0.2492126226425171, 0.29383403062820435, 0.32258570194244385], "cos_apical_negg": [0.00922171026468277, 0.04089028388261795, 0.06418897956609726], "cos_Ac_negg": [0.5633997321128845, 0.6470512747764587, 0.6645047068595886], "r_norm": [0.1495092213153839, 0.14799515902996063, 0.14744994044303894]}, {"step": 7000, "epoch": 14, "train_loss": 0.0003147691022604704, "cos_r_negg": [0.25122159719467163, 0.2936203181743622, 0.3225162625312805], "cos_innovation_negg": [0.25122159719467163, 0.2936203181743622, 0.3225162625312805], "cos_apical_negg": [0.011610345914959908, 0.042298395186662674, 0.06544257700443268], "cos_Ac_negg": [0.5647405385971069, 0.6474350690841675, 0.6659547686576843], "r_norm": [0.15288464725017548, 0.1513034999370575, 0.15089458227157593], "ldr": {"dL_sdil": -0.009760968387126923, "dL_bp": -0.01906360685825348, "ratio": 0.5120210702887511}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9736, "test_loss": 0.14643452949523925}], "final": {"test_acc": 0.9736, "test_loss": 0.14643452949523925, "wall_s": 23.490625381469727, "cos_r_negg": [0.24873177707195282, 0.2926154136657715, 0.3203858733177185], "cos_innovation_negg": [0.24873177707195282, 0.2926154136657715, 0.3203858733177185], "cos_apical_negg": [0.010511450469493866, 0.04232539236545563, 0.06358501315116882], "cos_Ac_negg": [0.5633581280708313, 0.6458847522735596, 0.6646045446395874]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p05_residual_s4.json b/results/nuis_ctrl_v1_rho0p05_residual_s4.json new file mode 100644 index 0000000..443cad9 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p05_residual_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.05, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p05_residual_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.012779329903423786, 0.029934005811810493, 0.00944238156080246], "cos_innovation_negg": [-0.012779329903423786, 0.029934005811810493, 0.00944238156080246], "cos_apical_negg": [-0.013300376012921333, 0.029353521764278412, 0.008677935227751732], "cos_Ac_negg": [-0.012779482640326023, 0.029933910816907883, 0.009442325681447983], "r_norm": [4.848395347595215, 4.855302810668945, 4.948003768920898], "ldr": {"dL_sdil": -0.0033469200134277344, "dL_bp": -0.2672088146209717, "ratio": 0.01252548505248694}}, {"step": 100, "epoch": 0, "train_loss": 0.19363561272621155, "cos_r_negg": [0.459269642829895, 0.3797130286693573, 0.31844401359558105], "cos_innovation_negg": [0.459269642829895, 0.3797130286693573, 0.31844401359558105], "cos_apical_negg": [0.1869918406009674, 0.18621602654457092, 0.17861896753311157], "cos_Ac_negg": [0.45944228768348694, 0.3798229694366455, 0.31835034489631653], "r_norm": [0.9167285561561584, 0.9265720844268799, 0.9415656924247742]}, {"step": 200, "epoch": 0, "train_loss": 0.25281593203544617, "cos_r_negg": [0.5016197562217712, 0.4379856586456299, 0.3615146577358246], "cos_innovation_negg": [0.5016197562217712, 0.4379856586456299, 0.3615146577358246], "cos_apical_negg": [0.19982387125492096, 0.19391566514968872, 0.18751326203346252], "cos_Ac_negg": [0.5017473101615906, 0.4381555914878845, 0.3611891567707062], "r_norm": [0.7675690650939941, 0.7717165946960449, 0.7897419929504395]}, {"step": 300, "epoch": 0, "train_loss": 0.24676823616027832, "cos_r_negg": [0.534898042678833, 0.46356427669525146, 0.3760718107223511], "cos_innovation_negg": [0.534898042678833, 0.46356427669525146, 0.3760718107223511], "cos_apical_negg": [0.17926794290542603, 0.18904438614845276, 0.1845228672027588], "cos_Ac_negg": [0.5352267026901245, 0.46348699927330017, 0.37584298849105835], "r_norm": [0.5503602027893066, 0.5532886385917664, 0.5686604976654053]}, {"step": 400, "epoch": 0, "train_loss": 0.15302714705467224, "cos_r_negg": [0.5326367020606995, 0.47829481959342957, 0.4119530916213989], "cos_innovation_negg": [0.5326367020606995, 0.47829481959342957, 0.4119530916213989], "cos_apical_negg": [0.16547957062721252, 0.17010702192783356, 0.17529134452342987], "cos_Ac_negg": [0.5327945947647095, 0.4782412350177765, 0.41187041997909546], "r_norm": [0.524154782295227, 0.5277701020240784, 0.5397800207138062]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9447, "test_loss": 0.18155341796875}, {"step": 500, "epoch": 1, "train_loss": 0.17874321341514587, "cos_r_negg": [0.5227363109588623, 0.5101392269134521, 0.4588015377521515], "cos_innovation_negg": [0.5227363109588623, 0.5101392269134521, 0.4588015377521515], "cos_apical_negg": [0.1373881697654724, 0.15714529156684875, 0.17961546778678894], "cos_Ac_negg": [0.522687554359436, 0.5101361274719238, 0.45864057540893555], "r_norm": [0.5000540018081665, 0.5062152147293091, 0.5183804035186768], "ldr": {"dL_sdil": -0.012512341141700745, "dL_bp": -0.01118496060371399, "ratio": 1.1186754772784802}}, {"step": 600, "epoch": 1, "train_loss": 0.06491975486278534, "cos_r_negg": [0.5474497675895691, 0.5192220211029053, 0.4468640387058258], "cos_innovation_negg": [0.5474497675895691, 0.5192220211029053, 0.4468640387058258], "cos_apical_negg": [0.1271270364522934, 0.15750424563884735, 0.1773270070552826], "cos_Ac_negg": [0.5476289391517639, 0.5195400714874268, 0.44661420583724976], "r_norm": [0.4415815472602844, 0.44529294967651367, 0.45522838830947876]}, {"step": 700, "epoch": 1, "train_loss": 0.198802649974823, "cos_r_negg": [0.5695801973342896, 0.5482631921768188, 0.4604000151157379], "cos_innovation_negg": [0.5695801973342896, 0.5482631921768188, 0.4604000151157379], "cos_apical_negg": [0.147748202085495, 0.16263194382190704, 0.17765286564826965], "cos_Ac_negg": [0.5699089765548706, 0.5480311512947083, 0.4600382149219513], "r_norm": [0.47886714339256287, 0.4833182990550995, 0.4965375065803528]}, {"step": 800, "epoch": 1, "train_loss": 0.15646713972091675, "cos_r_negg": [0.567284107208252, 0.5482035279273987, 0.46796345710754395], "cos_innovation_negg": [0.567284107208252, 0.5482035279273987, 0.46796345710754395], "cos_apical_negg": [0.12772102653980255, 0.159915030002594, 0.17331650853157043], "cos_Ac_negg": [0.5676268339157104, 0.5480576753616333, 0.46787577867507935], "r_norm": [0.3809143900871277, 0.3821704685688019, 0.39236980676651]}, {"step": 900, "epoch": 1, "train_loss": 0.18561142683029175, "cos_r_negg": [0.5868179798126221, 0.5653269290924072, 0.481930673122406], "cos_innovation_negg": [0.5868179798126221, 0.5653269290924072, 0.481930673122406], "cos_apical_negg": [0.11873948574066162, 0.16003139317035675, 0.17157092690467834], "cos_Ac_negg": [0.5868556499481201, 0.5651793479919434, 0.4816763401031494], "r_norm": [0.43814408779144287, 0.4417032301425934, 0.451862096786499]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9536, "test_loss": 0.15419194259643554}, {"step": 1000, "epoch": 2, "train_loss": 0.02181781455874443, "cos_r_negg": [0.5739259719848633, 0.5635170340538025, 0.48731517791748047], "cos_innovation_negg": [0.5739259719848633, 0.5635170340538025, 0.48731517791748047], "cos_apical_negg": [0.11029858887195587, 0.13616223633289337, 0.1503383219242096], "cos_Ac_negg": [0.5741055607795715, 0.563295841217041, 0.4872198700904846], "r_norm": [0.36802178621292114, 0.3702920079231262, 0.3779096007347107], "ldr": {"dL_sdil": -0.011937856674194336, "dL_bp": -0.010014474391937256, "ratio": 1.1920602327173169}}, {"step": 1100, "epoch": 2, "train_loss": 0.11275234818458557, "cos_r_negg": [0.5853288769721985, 0.5786828994750977, 0.502781867980957], "cos_innovation_negg": [0.5853288769721985, 0.5786828994750977, 0.502781867980957], "cos_apical_negg": [0.11679544299840927, 0.14709730446338654, 0.16490384936332703], "cos_Ac_negg": [0.5859564542770386, 0.5788313746452332, 0.5025363564491272], "r_norm": [0.3950566053390503, 0.3970186412334442, 0.40572869777679443]}, {"step": 1200, "epoch": 2, "train_loss": 0.1583356261253357, "cos_r_negg": [0.5809619426727295, 0.5775206089019775, 0.49697285890579224], "cos_innovation_negg": [0.5809619426727295, 0.5775206089019775, 0.49697285890579224], "cos_apical_negg": [0.09338440001010895, 0.13124938309192657, 0.1401596963405609], "cos_Ac_negg": [0.5822113752365112, 0.5778946280479431, 0.49724453687667847], "r_norm": [0.3299587368965149, 0.3310512900352478, 0.33843719959259033]}, {"step": 1300, "epoch": 2, "train_loss": 0.12771108746528625, "cos_r_negg": [0.6014107465744019, 0.6022679209709167, 0.5082502365112305], "cos_innovation_negg": [0.6014107465744019, 0.6022679209709167, 0.5082502365112305], "cos_apical_negg": [0.10280664265155792, 0.13716718554496765, 0.1483091562986374], "cos_Ac_negg": [0.6017036437988281, 0.6021094918251038, 0.5082778334617615], "r_norm": [0.35670411586761475, 0.3595981299877167, 0.368849515914917]}, {"step": 1400, "epoch": 2, "train_loss": 0.10537157952785492, "cos_r_negg": [0.5921329259872437, 0.5912002325057983, 0.5061360001564026], "cos_innovation_negg": [0.5921329259872437, 0.5912002325057983, 0.5061360001564026], "cos_apical_negg": [0.09046350419521332, 0.13055309653282166, 0.15138307213783264], "cos_Ac_negg": [0.5946567058563232, 0.5925124883651733, 0.506708562374115], "r_norm": [0.31271618604660034, 0.3149046301841736, 0.3246137499809265]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9631, "test_loss": 0.12175595703125}, {"step": 1500, "epoch": 3, "train_loss": 0.035476528108119965, "cos_r_negg": [0.5890349745750427, 0.6021519303321838, 0.5385955572128296], "cos_innovation_negg": [0.5890349745750427, 0.6021519303321838, 0.5385955572128296], "cos_apical_negg": [0.06733257323503494, 0.10770899802446365, 0.14317253232002258], "cos_Ac_negg": [0.5905673503875732, 0.6032276153564453, 0.5383012294769287], "r_norm": [0.2837359309196472, 0.28518205881118774, 0.2939888834953308], "ldr": {"dL_sdil": -0.007902510464191437, "dL_bp": -0.007275938987731934, "ratio": 1.086115548455804}}, {"step": 1600, "epoch": 3, "train_loss": 0.05675691366195679, "cos_r_negg": [0.5812368392944336, 0.6006582379341125, 0.5353538990020752], "cos_innovation_negg": [0.5812368392944336, 0.6006582379341125, 0.5353538990020752], "cos_apical_negg": [0.08494053781032562, 0.11340245604515076, 0.1384579837322235], "cos_Ac_negg": [0.5837069153785706, 0.6030481457710266, 0.5360864996910095], "r_norm": [0.28139248490333557, 0.2813023626804352, 0.2908490300178528]}, {"step": 1700, "epoch": 3, "train_loss": 0.03819672018289566, "cos_r_negg": [0.576766848564148, 0.5974439382553101, 0.5257833003997803], "cos_innovation_negg": [0.576766848564148, 0.5974439382553101, 0.5257833003997803], "cos_apical_negg": [0.07903841137886047, 0.11245925724506378, 0.1422579288482666], "cos_Ac_negg": [0.5808981657028198, 0.6002588272094727, 0.5256128311157227], "r_norm": [0.22938254475593567, 0.228070467710495, 0.23500370979309082]}, {"step": 1800, "epoch": 3, "train_loss": 0.012577962130308151, "cos_r_negg": [0.5960943698883057, 0.6038258671760559, 0.5261780619621277], "cos_innovation_negg": [0.5960943698883057, 0.6038258671760559, 0.5261780619621277], "cos_apical_negg": [0.07420073449611664, 0.11327838152647018, 0.12981820106506348], "cos_Ac_negg": [0.598205029964447, 0.6051718592643738, 0.5251649022102356], "r_norm": [0.23552271723747253, 0.23420119285583496, 0.24097025394439697]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9653, "test_loss": 0.1188214527130127}, {"step": 1900, "epoch": 4, "train_loss": 0.033263400197029114, "cos_r_negg": [0.5949555039405823, 0.6183372735977173, 0.5297161936759949], "cos_innovation_negg": [0.5949555039405823, 0.6183372735977173, 0.5297161936759949], "cos_apical_negg": [0.0582868829369545, 0.11855240166187286, 0.1389888972043991], "cos_Ac_negg": [0.5992276668548584, 0.6210401654243469, 0.5292882919311523], "r_norm": [0.25039803981781006, 0.2509022355079651, 0.25720369815826416]}, {"step": 2000, "epoch": 4, "train_loss": 0.07706649601459503, "cos_r_negg": [0.5977005362510681, 0.609539270401001, 0.5329980850219727], "cos_innovation_negg": [0.5977005362510681, 0.609539270401001, 0.5329980850219727], "cos_apical_negg": [0.0513223260641098, 0.10410377383232117, 0.1242242157459259], "cos_Ac_negg": [0.6040879487991333, 0.6144353151321411, 0.5341240167617798], "r_norm": [0.2246806025505066, 0.223198801279068, 0.23086339235305786], "ldr": {"dL_sdil": -0.005203373730182648, "dL_bp": -0.005424492061138153, "ratio": 0.9592370440470124}}, {"step": 2100, "epoch": 4, "train_loss": 0.027847032994031906, "cos_r_negg": [0.6074799299240112, 0.6233466267585754, 0.5568576455116272], "cos_innovation_negg": [0.6074799299240112, 0.6233466267585754, 0.5568576455116272], "cos_apical_negg": [0.06025214493274689, 0.10350442677736282, 0.1337289661169052], "cos_Ac_negg": [0.6121597290039062, 0.626561164855957, 0.5566967725753784], "r_norm": [0.24921327829360962, 0.24901887774467468, 0.25541913509368896]}, {"step": 2200, "epoch": 4, "train_loss": 0.018447523936629295, "cos_r_negg": [0.6021316051483154, 0.6244376301765442, 0.5686969757080078], "cos_innovation_negg": [0.6021316051483154, 0.6244376301765442, 0.5686969757080078], "cos_apical_negg": [0.057232726365327835, 0.09396456182003021, 0.13490691781044006], "cos_Ac_negg": [0.609283447265625, 0.6303319931030273, 0.5689502954483032], "r_norm": [0.2358730435371399, 0.23480519652366638, 0.24117043614387512]}, {"step": 2300, "epoch": 4, "train_loss": 0.17033937573432922, "cos_r_negg": [0.5807777643203735, 0.6194117069244385, 0.5654565095901489], "cos_innovation_negg": [0.5807777643203735, 0.6194117069244385, 0.5654565095901489], "cos_apical_negg": [0.06197572499513626, 0.11007210612297058, 0.1384003460407257], "cos_Ac_negg": [0.5930091738700867, 0.6267299056053162, 0.5684207677841187], "r_norm": [0.23120008409023285, 0.2296319454908371, 0.23754820227622986]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9679, "test_loss": 0.11396435432434082}, {"step": 2400, "epoch": 5, "train_loss": 0.07778006047010422, "cos_r_negg": [0.5961868762969971, 0.628020703792572, 0.5711195468902588], "cos_innovation_negg": [0.5961868762969971, 0.628020703792572, 0.5711195468902588], "cos_apical_negg": [0.05230245739221573, 0.08951733261346817, 0.12939926981925964], "cos_Ac_negg": [0.613296389579773, 0.6407097578048706, 0.5756195187568665], "r_norm": [0.24527998268604279, 0.24481268227100372, 0.25092095136642456]}, {"step": 2500, "epoch": 5, "train_loss": 0.03131214156746864, "cos_r_negg": [0.5886510610580444, 0.6179237365722656, 0.5504201054573059], "cos_innovation_negg": [0.5886510610580444, 0.6179237365722656, 0.5504201054573059], "cos_apical_negg": [0.053261227905750275, 0.09723541885614395, 0.13756129145622253], "cos_Ac_negg": [0.6131516695022583, 0.6393392086029053, 0.5581721067428589], "r_norm": [0.203288272023201, 0.20195026695728302, 0.20764616131782532], "ldr": {"dL_sdil": -0.005027107894420624, "dL_bp": -0.00556723028421402, "ratio": 0.9029818487435444}}, {"step": 2600, "epoch": 5, "train_loss": 0.05235353484749794, "cos_r_negg": [0.5919328927993774, 0.6215847730636597, 0.5753085613250732], "cos_innovation_negg": [0.5919328927993774, 0.6215847730636597, 0.5753085613250732], "cos_apical_negg": [0.03845927119255066, 0.09658487141132355, 0.1456366777420044], "cos_Ac_negg": [0.6172924041748047, 0.6442195773124695, 0.5827828645706177], "r_norm": [0.21641728281974792, 0.21532754600048065, 0.22124408185482025]}, {"step": 2700, "epoch": 5, "train_loss": 0.04534776508808136, "cos_r_negg": [0.5973405838012695, 0.6376732587814331, 0.5805200934410095], "cos_innovation_negg": [0.5973405838012695, 0.6376732587814331, 0.5805200934410095], "cos_apical_negg": [0.05731664597988129, 0.09314532577991486, 0.13641613721847534], "cos_Ac_negg": [0.6209036111831665, 0.6600388288497925, 0.5867773294448853], "r_norm": [0.2010490745306015, 0.199209526181221, 0.2040197104215622]}, {"step": 2800, "epoch": 5, "train_loss": 0.04737924411892891, "cos_r_negg": [0.587195873260498, 0.629308819770813, 0.5862987637519836], "cos_innovation_negg": [0.587195873260498, 0.629308819770813, 0.5862987637519836], "cos_apical_negg": [0.04214123636484146, 0.091066375374794, 0.13282956182956696], "cos_Ac_negg": [0.6158110499382019, 0.6564095616340637, 0.5958699584007263], "r_norm": [0.21488341689109802, 0.21427884697914124, 0.2195425182580948]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9702, "test_loss": 0.11145480041503907}, {"step": 2900, "epoch": 6, "train_loss": 0.02309112623333931, "cos_r_negg": [0.565677285194397, 0.6131430864334106, 0.5850650072097778], "cos_innovation_negg": [0.565677285194397, 0.6131430864334106, 0.5850650072097778], "cos_apical_negg": [0.050060540437698364, 0.08606230467557907, 0.13248473405838013], "cos_Ac_negg": [0.6040457487106323, 0.6533260345458984, 0.6031286120414734], "r_norm": [0.20840919017791748, 0.20701190829277039, 0.21399539709091187]}, {"step": 3000, "epoch": 6, "train_loss": 0.07823412120342255, "cos_r_negg": [0.5583109259605408, 0.6086455583572388, 0.5694288015365601], "cos_innovation_negg": [0.5583109259605408, 0.6086455583572388, 0.5694288015365601], "cos_apical_negg": [0.036664627492427826, 0.08008675277233124, 0.13559503853321075], "cos_Ac_negg": [0.6049589514732361, 0.6520075798034668, 0.5900802612304688], "r_norm": [0.19045023620128632, 0.18850593268871307, 0.19622398912906647], "ldr": {"dL_sdil": -0.005181856453418732, "dL_bp": -0.006142161786556244, "ratio": 0.8436535268023392}}, {"step": 3100, "epoch": 6, "train_loss": 0.013891670852899551, "cos_r_negg": [0.5546495914459229, 0.6070494651794434, 0.5782874822616577], "cos_innovation_negg": [0.5546495914459229, 0.6070494651794434, 0.5782874822616577], "cos_apical_negg": [0.03110378235578537, 0.07507039606571198, 0.12861861288547516], "cos_Ac_negg": [0.6110647916793823, 0.6585196256637573, 0.6041256189346313], "r_norm": [0.18141630291938782, 0.180147185921669, 0.18428057432174683]}, {"step": 3200, "epoch": 6, "train_loss": 0.022837044671177864, "cos_r_negg": [0.5567091703414917, 0.5884954333305359, 0.5748907327651978], "cos_innovation_negg": [0.5567091703414917, 0.5884954333305359, 0.5748907327651978], "cos_apical_negg": [0.03017181158065796, 0.07870163023471832, 0.1239771842956543], "cos_Ac_negg": [0.6206838488578796, 0.6521150469779968, 0.614775538444519], "r_norm": [0.2074543535709381, 0.2065643072128296, 0.21191434562206268]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9699, "test_loss": 0.12311956329345704}, {"step": 3300, "epoch": 7, "train_loss": 0.0072054145857691765, "cos_r_negg": [0.5515697002410889, 0.5856539011001587, 0.5689003467559814], "cos_innovation_negg": [0.5515697002410889, 0.5856539011001587, 0.5689003467559814], "cos_apical_negg": [0.0335095189511776, 0.07867659628391266, 0.12692901492118835], "cos_Ac_negg": [0.6189903616905212, 0.6502972841262817, 0.6095094084739685], "r_norm": [0.199436753988266, 0.1982872486114502, 0.20319654047489166]}, {"step": 3400, "epoch": 7, "train_loss": 0.023439813405275345, "cos_r_negg": [0.5239970684051514, 0.5703652501106262, 0.5639129877090454], "cos_innovation_negg": [0.5239970684051514, 0.5703652501106262, 0.5639129877090454], "cos_apical_negg": [0.03912169858813286, 0.08066577464342117, 0.13582327961921692], "cos_Ac_negg": [0.6044108271598816, 0.6493125557899475, 0.6172916293144226], "r_norm": [0.21733565628528595, 0.21749639511108398, 0.2214263528585434]}, {"step": 3500, "epoch": 7, "train_loss": 0.0055810222402215, "cos_r_negg": [0.506786048412323, 0.5491162538528442, 0.5527008175849915], "cos_innovation_negg": [0.506786048412323, 0.5491162538528442, 0.5527008175849915], "cos_apical_negg": [0.023315371945500374, 0.08084549754858017, 0.13188579678535461], "cos_Ac_negg": [0.6118475794792175, 0.6495540142059326, 0.6248893737792969], "r_norm": [0.17132240533828735, 0.16975370049476624, 0.17366693913936615], "ldr": {"dL_sdil": -0.0044611915946006775, "dL_bp": -0.005936257541179657, "ratio": 0.7515158437203091}}, {"step": 3600, "epoch": 7, "train_loss": 0.011079233139753342, "cos_r_negg": [0.503638505935669, 0.5509056448936462, 0.5584039688110352], "cos_innovation_negg": [0.503638505935669, 0.5509056448936462, 0.5584039688110352], "cos_apical_negg": [0.032642610371112823, 0.08205906301736832, 0.13160008192062378], "cos_Ac_negg": [0.6165440678596497, 0.6597976684570312, 0.6356155872344971], "r_norm": [0.1889066845178604, 0.18782877922058105, 0.19196265935897827]}, {"step": 3700, "epoch": 7, "train_loss": 0.019731123000383377, "cos_r_negg": [0.5026959776878357, 0.5492781400680542, 0.5546693801879883], "cos_innovation_negg": [0.5026959776878357, 0.5492781400680542, 0.5546693801879883], "cos_apical_negg": [0.028425056487321854, 0.07734984159469604, 0.12169761210680008], "cos_Ac_negg": [0.6090456247329712, 0.6525936126708984, 0.6372885704040527], "r_norm": [0.20464704930782318, 0.20316657423973083, 0.20876076817512512]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9714, "test_loss": 0.12079893531799317}, {"step": 3800, "epoch": 8, "train_loss": 0.005900415126234293, "cos_r_negg": [0.5039244294166565, 0.5404571294784546, 0.5490973591804504], "cos_innovation_negg": [0.5039244294166565, 0.5404571294784546, 0.5490973591804504], "cos_apical_negg": [0.03376132994890213, 0.07072678208351135, 0.12185116112232208], "cos_Ac_negg": [0.6261590719223022, 0.6571127772331238, 0.6360654830932617], "r_norm": [0.17702902853488922, 0.17459113895893097, 0.18020063638687134]}, {"step": 3900, "epoch": 8, "train_loss": 0.009754760190844536, "cos_r_negg": [0.4679676294326782, 0.5141180753707886, 0.524848222732544], "cos_innovation_negg": [0.4679676294326782, 0.5141180753707886, 0.524848222732544], "cos_apical_negg": [0.027421411126852036, 0.06862695515155792, 0.1141679435968399], "cos_Ac_negg": [0.6102022528648376, 0.6468222141265869, 0.63695228099823], "r_norm": [0.18532976508140564, 0.18423914909362793, 0.18915408849716187]}, {"step": 4000, "epoch": 8, "train_loss": 0.007884769700467587, "cos_r_negg": [0.4535374045372009, 0.4969760775566101, 0.5088882446289062], "cos_innovation_negg": [0.4535374045372009, 0.4969760775566101, 0.5088882446289062], "cos_apical_negg": [0.018973732367157936, 0.06523087620735168, 0.1068141981959343], "cos_Ac_negg": [0.6104272603988647, 0.6508997678756714, 0.6331040859222412], "r_norm": [0.17583787441253662, 0.17401285469532013, 0.1786840558052063], "ldr": {"dL_sdil": -0.006151571869850159, "dL_bp": -0.009266041219234467, "ratio": 0.6638834993611634}}, {"step": 4100, "epoch": 8, "train_loss": 0.0251475740224123, "cos_r_negg": [0.4754554033279419, 0.5207161903381348, 0.5318887233734131], "cos_innovation_negg": [0.4754554033279419, 0.5207161903381348, 0.5318887233734131], "cos_apical_negg": [0.021943775936961174, 0.07283858954906464, 0.11243140697479248], "cos_Ac_negg": [0.6093577742576599, 0.6566318273544312, 0.6427503228187561], "r_norm": [0.19794800877571106, 0.19629013538360596, 0.2009926587343216]}, {"step": 4200, "epoch": 8, "train_loss": 0.021346043795347214, "cos_r_negg": [0.435648113489151, 0.48645347356796265, 0.49681100249290466], "cos_innovation_negg": [0.435648113489151, 0.48645347356796265, 0.49681100249290466], "cos_apical_negg": [0.02471092715859413, 0.06892858445644379, 0.10212395340204239], "cos_Ac_negg": [0.6034913063049316, 0.6531990766525269, 0.6370819211006165], "r_norm": [0.1873244047164917, 0.18534408509731293, 0.19004282355308533]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9714, "test_loss": 0.13042741432189942}, {"step": 4300, "epoch": 9, "train_loss": 0.00207694573327899, "cos_r_negg": [0.4229526221752167, 0.4700314402580261, 0.48948386311531067], "cos_innovation_negg": [0.4229526221752167, 0.4700314402580261, 0.48948386311531067], "cos_apical_negg": [0.022869721055030823, 0.0754602700471878, 0.11757481098175049], "cos_Ac_negg": [0.6045072078704834, 0.6531006097793579, 0.6414310932159424], "r_norm": [0.19610975682735443, 0.19450807571411133, 0.19950783252716064]}, {"step": 4400, "epoch": 9, "train_loss": 0.0006989761022850871, "cos_r_negg": [0.43930789828300476, 0.48184335231781006, 0.4982673227787018], "cos_innovation_negg": [0.43930789828300476, 0.48184335231781006, 0.4982673227787018], "cos_apical_negg": [0.02033047005534172, 0.06314921379089355, 0.1044493168592453], "cos_Ac_negg": [0.6156895160675049, 0.660896360874176, 0.6490445137023926], "r_norm": [0.188154935836792, 0.1873258352279663, 0.19170327484607697]}, {"step": 4500, "epoch": 9, "train_loss": 0.0670974925160408, "cos_r_negg": [0.40810462832450867, 0.45823216438293457, 0.47662705183029175], "cos_innovation_negg": [0.40810462832450867, 0.45823216438293457, 0.47662705183029175], "cos_apical_negg": [0.013115651905536652, 0.06181355565786362, 0.10247022658586502], "cos_Ac_negg": [0.6016995906829834, 0.6549803018569946, 0.6481812000274658], "r_norm": [0.16602489352226257, 0.16475547850131989, 0.17002645134925842], "ldr": {"dL_sdil": -0.005957663059234619, "dL_bp": -0.009596183896064758, "ratio": 0.6208366913099448}}, {"step": 4600, "epoch": 9, "train_loss": 0.003053926629945636, "cos_r_negg": [0.413961261510849, 0.4633459150791168, 0.48455673456192017], "cos_innovation_negg": [0.413961261510849, 0.4633459150791168, 0.48455673456192017], "cos_apical_negg": [0.02628561109304428, 0.06421220302581787, 0.10565322637557983], "cos_Ac_negg": [0.5981574058532715, 0.6555895805358887, 0.6470040082931519], "r_norm": [0.17618034780025482, 0.1753236949443817, 0.18065184354782104]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9723, "test_loss": 0.13113348865509034}, {"step": 4700, "epoch": 10, "train_loss": 0.001840092591010034, "cos_r_negg": [0.3906187415122986, 0.4429139196872711, 0.4664008617401123], "cos_innovation_negg": [0.3906187415122986, 0.4429139196872711, 0.4664008617401123], "cos_apical_negg": [0.02556668035686016, 0.06279965490102768, 0.10503296554088593], "cos_Ac_negg": [0.6065284013748169, 0.6604180335998535, 0.6510452032089233], "r_norm": [0.1806732714176178, 0.18004316091537476, 0.18403521180152893]}, {"step": 4800, "epoch": 10, "train_loss": 0.002465236699208617, "cos_r_negg": [0.3616454601287842, 0.4148251712322235, 0.4383332133293152], "cos_innovation_negg": [0.3616454601287842, 0.4148251712322235, 0.4383332133293152], "cos_apical_negg": [0.019552316516637802, 0.06104365363717079, 0.1010805070400238], "cos_Ac_negg": [0.6002198457717896, 0.6596728563308716, 0.6473992466926575], "r_norm": [0.16908667981624603, 0.16726604104042053, 0.1718483567237854]}, {"step": 4900, "epoch": 10, "train_loss": 0.0009035018738359213, "cos_r_negg": [0.3674590587615967, 0.41435733437538147, 0.44010066986083984], "cos_innovation_negg": [0.3674590587615967, 0.41435733437538147, 0.44010066986083984], "cos_apical_negg": [0.016901742666959763, 0.0587710440158844, 0.10026903450489044], "cos_Ac_negg": [0.598545491695404, 0.6587402820587158, 0.6462540030479431], "r_norm": [0.16530188918113708, 0.16370832920074463, 0.16769784688949585]}, {"step": 5000, "epoch": 10, "train_loss": 0.001396470470353961, "cos_r_negg": [0.35372424125671387, 0.40318048000335693, 0.42876148223876953], "cos_innovation_negg": [0.35372424125671387, 0.40318048000335693, 0.42876148223876953], "cos_apical_negg": [0.015281818807125092, 0.06484194099903107, 0.10090282559394836], "cos_Ac_negg": [0.5906784534454346, 0.6562011241912842, 0.6522396206855774], "r_norm": [0.16465550661087036, 0.16371382772922516, 0.1678890585899353], "ldr": {"dL_sdil": -0.006827443838119507, "dL_bp": -0.011931560933589935, "ratio": 0.5722171538259315}}, {"step": 5100, "epoch": 10, "train_loss": 0.0065423608757555485, "cos_r_negg": [0.3494974970817566, 0.4002249836921692, 0.4248924255371094], "cos_innovation_negg": [0.3494974970817566, 0.4002249836921692, 0.4248924255371094], "cos_apical_negg": [0.010526479221880436, 0.060380127280950546, 0.09512849152088165], "cos_Ac_negg": [0.6005849242210388, 0.6609558463096619, 0.6511019468307495], "r_norm": [0.17341387271881104, 0.1718728244304657, 0.1758197844028473]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9719, "test_loss": 0.13702383298873902}, {"step": 5200, "epoch": 11, "train_loss": 0.0007825077045708895, "cos_r_negg": [0.352170467376709, 0.4004124104976654, 0.42119207978248596], "cos_innovation_negg": [0.352170467376709, 0.4004124104976654, 0.42119207978248596], "cos_apical_negg": [0.009563511237502098, 0.05999604985117912, 0.0949019342660904], "cos_Ac_negg": [0.6010615825653076, 0.6613147854804993, 0.6512768864631653], "r_norm": [0.18210366368293762, 0.18123450875282288, 0.18493232131004333]}, {"step": 5300, "epoch": 11, "train_loss": 0.001030321465805173, "cos_r_negg": [0.3405805230140686, 0.39221519231796265, 0.4158976376056671], "cos_innovation_negg": [0.3405805230140686, 0.39221519231796265, 0.4158976376056671], "cos_apical_negg": [0.006938896141946316, 0.05888422578573227, 0.09161244332790375], "cos_Ac_negg": [0.5965147018432617, 0.6610060334205627, 0.6546416282653809], "r_norm": [0.17633305490016937, 0.17482371628284454, 0.178761824965477]}, {"step": 5400, "epoch": 11, "train_loss": 0.0042540449649095535, "cos_r_negg": [0.31929224729537964, 0.37281617522239685, 0.39695513248443604], "cos_innovation_negg": [0.31929224729537964, 0.37281617522239685, 0.39695513248443604], "cos_apical_negg": [0.007814566604793072, 0.055944036692380905, 0.08859697729349136], "cos_Ac_negg": [0.5906990766525269, 0.6589862108230591, 0.6522208452224731], "r_norm": [0.1626206636428833, 0.16141757369041443, 0.16495022177696228]}, {"step": 5500, "epoch": 11, "train_loss": 0.00109402637463063, "cos_r_negg": [0.3236708641052246, 0.37145695090293884, 0.3936476707458496], "cos_innovation_negg": [0.3236708641052246, 0.37145695090293884, 0.3936476707458496], "cos_apical_negg": [0.009464427828788757, 0.05492686852812767, 0.08791422843933105], "cos_Ac_negg": [0.5948855876922607, 0.6594706773757935, 0.6522601842880249], "r_norm": [0.16877172887325287, 0.16703590750694275, 0.17109626531600952], "ldr": {"dL_sdil": -0.0071260035037994385, "dL_bp": -0.01254795491695404, "ratio": 0.5679015864307269}}, {"step": 5600, "epoch": 11, "train_loss": 0.0003725747228600085, "cos_r_negg": [0.30669617652893066, 0.3494783639907837, 0.3695250451564789], "cos_innovation_negg": [0.30669617652893066, 0.3494783639907837, 0.3695250451564789], "cos_apical_negg": [0.008274903520941734, 0.05524459481239319, 0.08129556477069855], "cos_Ac_negg": [0.5895830988883972, 0.6564111113548279, 0.6514990329742432], "r_norm": [0.16499063372612, 0.16279491782188416, 0.1670016497373581]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9722, "test_loss": 0.13796027421951293}, {"step": 5700, "epoch": 12, "train_loss": 0.00023269107623491436, "cos_r_negg": [0.3141196668148041, 0.3611403703689575, 0.384514719247818], "cos_innovation_negg": [0.3141196668148041, 0.3611403703689575, 0.384514719247818], "cos_apical_negg": [0.006977427750825882, 0.05657617002725601, 0.08557741343975067], "cos_Ac_negg": [0.5944322943687439, 0.6608906984329224, 0.655097484588623], "r_norm": [0.16944685578346252, 0.16728347539901733, 0.17163588106632233]}, {"step": 5800, "epoch": 12, "train_loss": 0.00024193685385398567, "cos_r_negg": [0.3005027174949646, 0.3442278504371643, 0.36499232053756714], "cos_innovation_negg": [0.3005027174949646, 0.3442278504371643, 0.36499232053756714], "cos_apical_negg": [0.006468859501183033, 0.053578704595565796, 0.07757032662630081], "cos_Ac_negg": [0.584417462348938, 0.654517650604248, 0.6532986164093018], "r_norm": [0.16966773569583893, 0.1676851212978363, 0.17237864434719086]}, {"step": 5900, "epoch": 12, "train_loss": 0.0008451087051071227, "cos_r_negg": [0.2991112470626831, 0.3457573652267456, 0.36647093296051025], "cos_innovation_negg": [0.2991112470626831, 0.3457573652267456, 0.36647093296051025], "cos_apical_negg": [0.005139848217368126, 0.0564047209918499, 0.08021765947341919], "cos_Ac_negg": [0.5839756727218628, 0.655193567276001, 0.6558170318603516], "r_norm": [0.17006580531597137, 0.16849187016487122, 0.17286396026611328]}, {"step": 6000, "epoch": 12, "train_loss": 0.000429863459430635, "cos_r_negg": [0.2971387505531311, 0.34047025442123413, 0.3639088273048401], "cos_innovation_negg": [0.2971387505531311, 0.34047025442123413, 0.3639088273048401], "cos_apical_negg": [0.006260121241211891, 0.05526921898126602, 0.08228494971990585], "cos_Ac_negg": [0.5862312316894531, 0.6530728340148926, 0.6545469760894775], "r_norm": [0.16694281995296478, 0.16536740958690643, 0.16937360167503357], "ldr": {"dL_sdil": -0.007939636707305908, "dL_bp": -0.014572396874427795, "ratio": 0.5448408230796054}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9722, "test_loss": 0.1420075569152832}, {"step": 6100, "epoch": 13, "train_loss": 0.00045817336649633944, "cos_r_negg": [0.29320865869522095, 0.3298850655555725, 0.3504391610622406], "cos_innovation_negg": [0.29320865869522095, 0.3298850655555725, 0.3504391610622406], "cos_apical_negg": [0.007446726784110069, 0.05385761708021164, 0.07755032181739807], "cos_Ac_negg": [0.5862466096878052, 0.654208779335022, 0.6529350280761719], "r_norm": [0.1695239543914795, 0.16792729496955872, 0.17198199033737183]}, {"step": 6200, "epoch": 13, "train_loss": 0.0005318940966390073, "cos_r_negg": [0.28840914368629456, 0.32514163851737976, 0.3477312922477722], "cos_innovation_negg": [0.28840914368629456, 0.32514163851737976, 0.3477312922477722], "cos_apical_negg": [0.009004191495478153, 0.053208671510219574, 0.07626290619373322], "cos_Ac_negg": [0.5882088541984558, 0.6548550128936768, 0.6557705402374268], "r_norm": [0.17259481549263, 0.1705859899520874, 0.17510265111923218]}, {"step": 6300, "epoch": 13, "train_loss": 0.0020512929186224937, "cos_r_negg": [0.2850475311279297, 0.3188374638557434, 0.3398038148880005], "cos_innovation_negg": [0.2850475311279297, 0.3188374638557434, 0.3398038148880005], "cos_apical_negg": [0.008397301658987999, 0.05286361277103424, 0.07560718804597855], "cos_Ac_negg": [0.5849769115447998, 0.6537105441093445, 0.6546311974525452], "r_norm": [0.16976244747638702, 0.16809657216072083, 0.17217665910720825]}, {"step": 6400, "epoch": 13, "train_loss": 0.00027335615595802665, "cos_r_negg": [0.28622734546661377, 0.3215796947479248, 0.34399476647377014], "cos_innovation_negg": [0.28622734546661377, 0.3215796947479248, 0.34399476647377014], "cos_apical_negg": [0.007981441915035248, 0.053553562611341476, 0.0770130604505539], "cos_Ac_negg": [0.5863303542137146, 0.6551163196563721, 0.6560806035995483], "r_norm": [0.1701076626777649, 0.16836421191692352, 0.17273661494255066]}, {"step": 6500, "epoch": 13, "train_loss": 0.0001373695704387501, "cos_r_negg": [0.2824729084968567, 0.31745219230651855, 0.3403700590133667], "cos_innovation_negg": [0.2824729084968567, 0.31745219230651855, 0.3403700590133667], "cos_apical_negg": [0.009321839548647404, 0.05357018858194351, 0.07671408355236053], "cos_Ac_negg": [0.5849481821060181, 0.6539113521575928, 0.6556531190872192], "r_norm": [0.1614467054605484, 0.15989987552165985, 0.16377681493759155], "ldr": {"dL_sdil": -0.007981009781360626, "dL_bp": -0.01510775089263916, "ratio": 0.528272529648947}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9719, "test_loss": 0.14431319007873536}, {"step": 6600, "epoch": 14, "train_loss": 0.00011856538185384125, "cos_r_negg": [0.27665117383003235, 0.3105822801589966, 0.334550678730011], "cos_innovation_negg": [0.27665117383003235, 0.3105822801589966, 0.334550678730011], "cos_apical_negg": [0.007295699790120125, 0.05206109210848808, 0.07550030946731567], "cos_Ac_negg": [0.5844250917434692, 0.6526857614517212, 0.656484842300415], "r_norm": [0.16773851215839386, 0.16588163375854492, 0.16997557878494263]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008524881559424102, "cos_r_negg": [0.2749953866004944, 0.31169018149375916, 0.33443671464920044], "cos_innovation_negg": [0.2749953866004944, 0.31169018149375916, 0.33443671464920044], "cos_apical_negg": [0.007936772890388966, 0.05364764854311943, 0.0748809427022934], "cos_Ac_negg": [0.579275369644165, 0.6512957811355591, 0.656309187412262], "r_norm": [0.16454872488975525, 0.16312849521636963, 0.16714921593666077]}, {"step": 6800, "epoch": 14, "train_loss": 0.0016292296350002289, "cos_r_negg": [0.26679256558418274, 0.3007053732872009, 0.3244709372520447], "cos_innovation_negg": [0.26679256558418274, 0.3007053732872009, 0.3244709372520447], "cos_apical_negg": [0.0077690621837973595, 0.05183498561382294, 0.07318760454654694], "cos_Ac_negg": [0.58195960521698, 0.6518175005912781, 0.6554392576217651], "r_norm": [0.16643011569976807, 0.16466417908668518, 0.16892865300178528]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006684634136036038, "cos_r_negg": [0.26337456703186035, 0.2981535792350769, 0.32045748829841614], "cos_innovation_negg": [0.26337456703186035, 0.2981535792350769, 0.32045748829841614], "cos_apical_negg": [0.007672143168747425, 0.05288967117667198, 0.07224975526332855], "cos_Ac_negg": [0.5826812982559204, 0.6537188291549683, 0.656615138053894], "r_norm": [0.16910724341869354, 0.1671365648508072, 0.1714499294757843]}, {"step": 7000, "epoch": 14, "train_loss": 0.0012496009003371, "cos_r_negg": [0.2641991078853607, 0.2971504330635071, 0.3216572403907776], "cos_innovation_negg": [0.2641991078853607, 0.2971504330635071, 0.3216572403907776], "cos_apical_negg": [0.00854979082942009, 0.051634564995765686, 0.0736464112997055], "cos_Ac_negg": [0.5844832062721252, 0.6534990072250366, 0.6575264930725098], "r_norm": [0.16546377539634705, 0.16344332695007324, 0.16770461201667786], "ldr": {"dL_sdil": -0.008243657648563385, "dL_bp": -0.016118310391902924, "ratio": 0.5114467613618242}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9721, "test_loss": 0.14751017742156983}], "final": {"test_acc": 0.9721, "test_loss": 0.14751017742156983, "wall_s": 22.025368452072144, "cos_r_negg": [0.260580837726593, 0.29289305210113525, 0.31671878695487976], "cos_innovation_negg": [0.260580837726593, 0.29289305210113525, 0.31671878695487976], "cos_apical_negg": [0.009301045909523964, 0.0510302409529686, 0.07210469245910645], "cos_Ac_negg": [0.5810582041740417, 0.6503059267997742, 0.6564116477966309]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_matched_s0.json b/results/nuis_ctrl_v1_rho0p2_matched_s0.json new file mode 100644 index 0000000..03b1486 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_matched_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_matched_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.016910701990127563, -0.02604541927576065, 0.030188513919711113], "cos_innovation_negg": [-0.016985144466161728, -0.02644510567188263, 0.03330337256193161], "cos_apical_negg": [-0.016910701990127563, -0.02604541927576065, 0.030188513919711113], "cos_Ac_negg": [-0.01698516122996807, -0.026445209980010986, 0.03330322355031967], "r_norm": [4.896961212158203, 4.825404167175293, 4.822786331176758], "ldr": {"dL_sdil": -0.00569915771484375, "dL_bp": -0.217299222946167, "ratio": 0.026227234674721506}}, {"step": 100, "epoch": 0, "train_loss": 0.3671969175338745, "cos_r_negg": [0.21015483140945435, 0.23888519406318665, 0.42397183179855347], "cos_innovation_negg": [0.3846322298049927, 0.38752883672714233, 0.7821890115737915], "cos_apical_negg": [0.21015483140945435, 0.23888519406318665, 0.42397183179855347], "cos_Ac_negg": [0.3845224380493164, 0.3874472379684448, 0.7821180820465088], "r_norm": [1.3998701572418213, 1.37485933303833, 1.3636449575424194]}, {"step": 200, "epoch": 0, "train_loss": 0.436411052942276, "cos_r_negg": [0.16561050713062286, 0.18619100749492645, 0.31251081824302673], "cos_innovation_negg": [0.37013736367225647, 0.38426244258880615, 0.6920689940452576], "cos_apical_negg": [0.16561050713062286, 0.18619099259376526, 0.3125108480453491], "cos_Ac_negg": [0.36999818682670593, 0.3841216266155243, 0.6919796466827393], "r_norm": [1.0953254699707031, 1.0818583965301514, 1.07880699634552]}, {"step": 300, "epoch": 0, "train_loss": 0.5078734755516052, "cos_r_negg": [0.13191697001457214, 0.16002486646175385, 0.3476477265357971], "cos_innovation_negg": [0.39260271191596985, 0.39785540103912354, 0.7923102378845215], "cos_apical_negg": [0.13191695511341095, 0.16002486646175385, 0.3476477265357971], "cos_Ac_negg": [0.3924560248851776, 0.3977569043636322, 0.7923903465270996], "r_norm": [0.9665123224258423, 0.9538863897323608, 0.9485065340995789]}, {"step": 400, "epoch": 0, "train_loss": 0.30207210779190063, "cos_r_negg": [0.13585889339447021, 0.15289762616157532, 0.22519588470458984], "cos_innovation_negg": [0.3580482602119446, 0.37615275382995605, 0.7226695418357849], "cos_apical_negg": [0.13585889339447021, 0.1528976410627365, 0.22519588470458984], "cos_Ac_negg": [0.35783514380455017, 0.37602895498275757, 0.7226627469062805], "r_norm": [1.0226716995239258, 1.0126649141311646, 1.004954218864441]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9179, "test_loss": 0.32707307434082034}, {"step": 500, "epoch": 1, "train_loss": 0.46707284450531006, "cos_r_negg": [0.148612380027771, 0.16521328687667847, 0.24070119857788086], "cos_innovation_negg": [0.3836321234703064, 0.38877925276756287, 0.6985004544258118], "cos_apical_negg": [0.148612380027771, 0.16521328687667847, 0.24070119857788086], "cos_Ac_negg": [0.3835386633872986, 0.38876867294311523, 0.698553204536438], "r_norm": [1.0348248481750488, 1.0256242752075195, 1.0130565166473389], "ldr": {"dL_sdil": -0.034368276596069336, "dL_bp": -0.03947359323501587, "ratio": 0.8706650137333392}}, {"step": 600, "epoch": 1, "train_loss": 0.1800658404827118, "cos_r_negg": [0.11802736669778824, 0.15870866179466248, 0.25164997577667236], "cos_innovation_negg": [0.35553085803985596, 0.40532630681991577, 0.7008638381958008], "cos_apical_negg": [0.11802736669778824, 0.15870866179466248, 0.25164997577667236], "cos_Ac_negg": [0.35541051626205444, 0.40529096126556396, 0.7008581757545471], "r_norm": [0.9126423001289368, 0.9019166827201843, 0.8938847780227661]}, {"step": 700, "epoch": 1, "train_loss": 0.27306556701660156, "cos_r_negg": [0.15432460606098175, 0.15079769492149353, 0.17088043689727783], "cos_innovation_negg": [0.3698684573173523, 0.39861229062080383, 0.694570779800415], "cos_apical_negg": [0.15432460606098175, 0.15079769492149353, 0.17088043689727783], "cos_Ac_negg": [0.3698160648345947, 0.3986052870750427, 0.6945176720619202], "r_norm": [0.8283841609954834, 0.8145154714584351, 0.8125600218772888]}, {"step": 800, "epoch": 1, "train_loss": 0.2946692109107971, "cos_r_negg": [0.10761032998561859, 0.10931065678596497, 0.2872355580329895], "cos_innovation_negg": [0.35262224078178406, 0.37932413816452026, 0.6317192316055298], "cos_apical_negg": [0.10761033743619919, 0.10931065678596497, 0.2872355580329895], "cos_Ac_negg": [0.352547287940979, 0.3792979419231415, 0.6315848231315613], "r_norm": [0.7779655456542969, 0.7659631371498108, 0.7699404358863831]}, {"step": 900, "epoch": 1, "train_loss": 0.2935892343521118, "cos_r_negg": [0.12415626645088196, 0.14709091186523438, 0.2504137456417084], "cos_innovation_negg": [0.37641313672065735, 0.43666723370552063, 0.6433188915252686], "cos_apical_negg": [0.12415626645088196, 0.14709091186523438, 0.2504137456417084], "cos_Ac_negg": [0.37636154890060425, 0.43662896752357483, 0.6431876420974731], "r_norm": [0.8178730607032776, 0.8104053139686584, 0.8023651838302612]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9218, "test_loss": 0.30185379486083985}, {"step": 1000, "epoch": 2, "train_loss": 0.20073287189006805, "cos_r_negg": [0.13279801607131958, 0.1912076473236084, 0.23135265707969666], "cos_innovation_negg": [0.36067524552345276, 0.40672019124031067, 0.6943851113319397], "cos_apical_negg": [0.13279801607131958, 0.1912076771259308, 0.23135265707969666], "cos_Ac_negg": [0.3606714606285095, 0.40666139125823975, 0.694292426109314], "r_norm": [0.9124011993408203, 0.9028640985488892, 0.8986786603927612], "ldr": {"dL_sdil": -0.049310892820358276, "dL_bp": -0.07275325059890747, "ratio": 0.6777826751991034}}, {"step": 1100, "epoch": 2, "train_loss": 0.33795085549354553, "cos_r_negg": [0.10380343347787857, 0.15223342180252075, 0.08692482113838196], "cos_innovation_negg": [0.3483837842941284, 0.3704032301902771, 0.6581887006759644], "cos_apical_negg": [0.10380344092845917, 0.15223342180252075, 0.08692481368780136], "cos_Ac_negg": [0.3483674228191376, 0.3703528046607971, 0.6578901410102844], "r_norm": [0.6779090166091919, 0.6698195934295654, 0.6667187213897705]}, {"step": 1200, "epoch": 2, "train_loss": 0.2874489426612854, "cos_r_negg": [0.11446413397789001, 0.1141526848077774, 0.1596776396036148], "cos_innovation_negg": [0.3667020797729492, 0.3804623782634735, 0.6569598913192749], "cos_apical_negg": [0.11446413397789001, 0.1141526848077774, 0.159677654504776], "cos_Ac_negg": [0.36661791801452637, 0.3803761601448059, 0.6562357544898987], "r_norm": [0.6565137505531311, 0.6446558237075806, 0.6440473794937134]}, {"step": 1300, "epoch": 2, "train_loss": 0.31487780809402466, "cos_r_negg": [0.11035647243261337, 0.14371353387832642, 0.23060396313667297], "cos_innovation_negg": [0.41421306133270264, 0.4204883873462677, 0.6779090762138367], "cos_apical_negg": [0.11035646498203278, 0.14371353387832642, 0.23060396313667297], "cos_Ac_negg": [0.4140692949295044, 0.4204024076461792, 0.6777994632720947], "r_norm": [0.7519258856773376, 0.7431415915489197, 0.740390419960022]}, {"step": 1400, "epoch": 2, "train_loss": 0.32602959871292114, "cos_r_negg": [0.10326863825321198, 0.1437813639640808, 0.23308955132961273], "cos_innovation_negg": [0.3792704939842224, 0.4069952666759491, 0.5764225721359253], "cos_apical_negg": [0.10326863825321198, 0.14378134906291962, 0.23308953642845154], "cos_Ac_negg": [0.3790837228298187, 0.4069123864173889, 0.5764555931091309], "r_norm": [0.7261743545532227, 0.7197409272193909, 0.7132318019866943]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9414, "test_loss": 0.2383779281616211}, {"step": 1500, "epoch": 3, "train_loss": 0.19006186723709106, "cos_r_negg": [0.07787270843982697, 0.10619936883449554, 0.23018766939640045], "cos_innovation_negg": [0.32801008224487305, 0.37394922971725464, 0.6187731027603149], "cos_apical_negg": [0.07787270843982697, 0.10619936883449554, 0.23018768429756165], "cos_Ac_negg": [0.32792049646377563, 0.37382248044013977, 0.6178521513938904], "r_norm": [0.6307463645935059, 0.6233481168746948, 0.6202783584594727], "ldr": {"dL_sdil": -0.010799944400787354, "dL_bp": -0.0258035808801651, "ratio": 0.4185444047841104}}, {"step": 1600, "epoch": 3, "train_loss": 0.17900805175304413, "cos_r_negg": [0.05508226901292801, 0.07156813144683838, 0.1848665475845337], "cos_innovation_negg": [0.30129316449165344, 0.3596057891845703, 0.5015495419502258], "cos_apical_negg": [0.05508226528763771, 0.07156813144683838, 0.1848665475845337], "cos_Ac_negg": [0.30130165815353394, 0.35960403084754944, 0.5015653371810913], "r_norm": [0.6009255647659302, 0.5920724272727966, 0.5909700989723206]}, {"step": 1700, "epoch": 3, "train_loss": 0.09935538470745087, "cos_r_negg": [0.07471838593482971, 0.09182325750589371, 0.2272641807794571], "cos_innovation_negg": [0.33583492040634155, 0.38933977484703064, 0.5807477235794067], "cos_apical_negg": [0.07471838593482971, 0.09182325750589371, 0.2272641807794571], "cos_Ac_negg": [0.33571657538414, 0.3892342150211334, 0.5807944536209106], "r_norm": [0.5559509992599487, 0.5467633605003357, 0.5448875427246094]}, {"step": 1800, "epoch": 3, "train_loss": 0.11316844075918198, "cos_r_negg": [0.10128222405910492, 0.11084297299385071, 0.26183804869651794], "cos_innovation_negg": [0.30644533038139343, 0.38052183389663696, 0.7224230766296387], "cos_apical_negg": [0.10128222405910492, 0.11084297299385071, 0.26183804869651794], "cos_Ac_negg": [0.30615824460983276, 0.3805667459964752, 0.722957968711853], "r_norm": [0.7624257802963257, 0.7597299814224243, 0.7522482872009277]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9452, "test_loss": 0.22861594848632813}, {"step": 1900, "epoch": 4, "train_loss": 0.18404515087604523, "cos_r_negg": [0.07403753697872162, 0.08317174762487411, 0.17718476057052612], "cos_innovation_negg": [0.3081355690956116, 0.3828663229942322, 0.7042200565338135], "cos_apical_negg": [0.07403753697872162, 0.08317174017429352, 0.17718476057052612], "cos_Ac_negg": [0.30808889865875244, 0.38281139731407166, 0.7038575410842896], "r_norm": [0.4736280143260956, 0.46871525049209595, 0.4676632583141327]}, {"step": 2000, "epoch": 4, "train_loss": 0.19394999742507935, "cos_r_negg": [0.06783159077167511, 0.1057671457529068, 0.25194603204727173], "cos_innovation_negg": [0.2857992649078369, 0.3521230220794678, 0.7737895250320435], "cos_apical_negg": [0.06783159077167511, 0.1057671457529068, 0.25194603204727173], "cos_Ac_negg": [0.28575652837753296, 0.3520244061946869, 0.77375328540802], "r_norm": [0.5666619539260864, 0.5600080490112305, 0.559309720993042], "ldr": {"dL_sdil": -0.02102816104888916, "dL_bp": -0.039189547300338745, "ratio": 0.5365757580136017}}, {"step": 2100, "epoch": 4, "train_loss": 0.18153876066207886, "cos_r_negg": [0.06017449498176575, 0.10495573282241821, 0.17399781942367554], "cos_innovation_negg": [0.32202860713005066, 0.3844048082828522, 0.7080654501914978], "cos_apical_negg": [0.06017449498176575, 0.10495573282241821, 0.17399781942367554], "cos_Ac_negg": [0.3219680190086365, 0.384155809879303, 0.7084072828292847], "r_norm": [0.5235640406608582, 0.5207656621932983, 0.519165575504303]}, {"step": 2200, "epoch": 4, "train_loss": 0.1294316202402115, "cos_r_negg": [0.061022695153951645, 0.07286687195301056, 0.20322400331497192], "cos_innovation_negg": [0.293312668800354, 0.3381996154785156, 0.7668923139572144], "cos_apical_negg": [0.06102269887924194, 0.07286687195301056, 0.20322400331497192], "cos_Ac_negg": [0.2932765483856201, 0.3381524980068207, 0.7671206593513489], "r_norm": [0.5442402362823486, 0.5404903888702393, 0.5369423627853394]}, {"step": 2300, "epoch": 4, "train_loss": 0.33430320024490356, "cos_r_negg": [0.06579358875751495, 0.07947492599487305, 0.18638545274734497], "cos_innovation_negg": [0.33113187551498413, 0.37937575578689575, 0.7933905124664307], "cos_apical_negg": [0.06579358875751495, 0.07947491109371185, 0.18638545274734497], "cos_Ac_negg": [0.3310801088809967, 0.3793811500072479, 0.7936273813247681], "r_norm": [0.49536433815956116, 0.49219658970832825, 0.48724564909935]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9444, "test_loss": 0.2452650604248047}, {"step": 2400, "epoch": 5, "train_loss": 0.22562149167060852, "cos_r_negg": [0.06354087591171265, 0.09185168892145157, 0.1689814031124115], "cos_innovation_negg": [0.28300565481185913, 0.3377803564071655, 0.7773607969284058], "cos_apical_negg": [0.06354087591171265, 0.09185168147087097, 0.1689814180135727], "cos_Ac_negg": [0.2828659415245056, 0.3375649154186249, 0.7774609327316284], "r_norm": [0.46942460536956787, 0.46887916326522827, 0.46491581201553345]}, {"step": 2500, "epoch": 5, "train_loss": 0.1367867887020111, "cos_r_negg": [0.05020769312977791, 0.06341016292572021, 0.18395891785621643], "cos_innovation_negg": [0.30209749937057495, 0.3488248586654663, 0.7262640595436096], "cos_apical_negg": [0.05020769312977791, 0.06341016292572021, 0.18395893275737762], "cos_Ac_negg": [0.3020341396331787, 0.3488159477710724, 0.7264837026596069], "r_norm": [0.44485294818878174, 0.43907976150512695, 0.4399731457233429], "ldr": {"dL_sdil": -0.017463982105255127, "dL_bp": -0.03445836901664734, "ratio": 0.5068139498076076}}, {"step": 2600, "epoch": 5, "train_loss": 0.1583840399980545, "cos_r_negg": [0.08095865696668625, 0.0947069376707077, 0.21625381708145142], "cos_innovation_negg": [0.32949137687683105, 0.40221846103668213, 0.8347949981689453], "cos_apical_negg": [0.08095865696668625, 0.0947069376707077, 0.21625381708145142], "cos_Ac_negg": [0.3294947147369385, 0.4021814167499542, 0.8353608846664429], "r_norm": [0.47671666741371155, 0.4755612909793854, 0.47085919976234436]}, {"step": 2700, "epoch": 5, "train_loss": 0.2837687134742737, "cos_r_negg": [0.07245595753192902, 0.037062741816043854, 0.18054525554180145], "cos_innovation_negg": [0.29242581129074097, 0.36796140670776367, 0.7281601428985596], "cos_apical_negg": [0.07245596498250961, 0.037062741816043854, 0.18054527044296265], "cos_Ac_negg": [0.29233303666114807, 0.36800551414489746, 0.727829098701477], "r_norm": [0.47226056456565857, 0.46763715147972107, 0.4660550355911255]}, {"step": 2800, "epoch": 5, "train_loss": 0.33337318897247314, "cos_r_negg": [0.07551109045743942, 0.07199983298778534, 0.26270222663879395], "cos_innovation_negg": [0.3481215238571167, 0.3945929706096649, 0.7918924689292908], "cos_apical_negg": [0.07551109045743942, 0.07199983298778534, 0.2627022862434387], "cos_Ac_negg": [0.3480343520641327, 0.3945924639701843, 0.7921533584594727], "r_norm": [0.47214239835739136, 0.46911585330963135, 0.4644724130630493]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9463, "test_loss": 0.22101455764770508}, {"step": 2900, "epoch": 6, "train_loss": 0.0703803226351738, "cos_r_negg": [0.06519763916730881, 0.053710028529167175, 0.18359032273292542], "cos_innovation_negg": [0.3075125813484192, 0.3686094880104065, 0.7685954570770264], "cos_apical_negg": [0.06519763916730881, 0.053710028529167175, 0.18359030783176422], "cos_Ac_negg": [0.30725955963134766, 0.36857855319976807, 0.7703044414520264], "r_norm": [0.4364830255508423, 0.4318782389163971, 0.428119957447052]}, {"step": 3000, "epoch": 6, "train_loss": 0.3784041404724121, "cos_r_negg": [0.04223143309354782, 0.07707555592060089, 0.27504706382751465], "cos_innovation_negg": [0.32646453380584717, 0.391116738319397, 0.7765461206436157], "cos_apical_negg": [0.04223143309354782, 0.07707555592060089, 0.27504706382751465], "cos_Ac_negg": [0.32593685388565063, 0.391015887260437, 0.7795013189315796], "r_norm": [0.45033854246139526, 0.446958065032959, 0.44584330916404724], "ldr": {"dL_sdil": -0.00955994427204132, "dL_bp": -0.027794882655143738, "ratio": 0.3439462001208396}}, {"step": 3100, "epoch": 6, "train_loss": 0.22425976395606995, "cos_r_negg": [0.05647234618663788, 0.07435258477926254, 0.2162187695503235], "cos_innovation_negg": [0.3371387720108032, 0.37014251947402954, 0.7533335089683533], "cos_apical_negg": [0.05647234618663788, 0.07435258477926254, 0.2162187695503235], "cos_Ac_negg": [0.3370808959007263, 0.3700496256351471, 0.754345178604126], "r_norm": [0.40786993503570557, 0.4045412242412567, 0.40224242210388184]}, {"step": 3200, "epoch": 6, "train_loss": 0.1918620467185974, "cos_r_negg": [0.058074817061424255, 0.08640265464782715, 0.23449644446372986], "cos_innovation_negg": [0.3056862950325012, 0.39892852306365967, 0.8260382413864136], "cos_apical_negg": [0.058074817061424255, 0.08640265464782715, 0.23449644446372986], "cos_Ac_negg": [0.3056592047214508, 0.39897415041923523, 0.8269335031509399], "r_norm": [0.46215933561325073, 0.45546603202819824, 0.45650917291641235]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9472, "test_loss": 0.2610755874633789}, {"step": 3300, "epoch": 7, "train_loss": 0.19521722197532654, "cos_r_negg": [0.07807251811027527, 0.12491660565137863, -0.03461514413356781], "cos_innovation_negg": [0.27619075775146484, 0.36475229263305664, 0.8042594790458679], "cos_apical_negg": [0.07807251811027527, 0.12491659820079803, -0.03461514785885811], "cos_Ac_negg": [0.2761938273906708, 0.364795982837677, 0.8044165968894958], "r_norm": [0.6243636608123779, 0.6147435903549194, 0.6135791540145874]}, {"step": 3400, "epoch": 7, "train_loss": 0.25325125455856323, "cos_r_negg": [0.07746362686157227, 0.06173750385642052, -0.05358889326453209], "cos_innovation_negg": [0.29400771856307983, 0.3629285991191864, 0.7421488761901855], "cos_apical_negg": [0.07746363431215286, 0.06173750385642052, -0.053588904440402985], "cos_Ac_negg": [0.2941010594367981, 0.36311179399490356, 0.7437472343444824], "r_norm": [0.4406527876853943, 0.43645551800727844, 0.43288204073905945]}, {"step": 3500, "epoch": 7, "train_loss": 0.13610272109508514, "cos_r_negg": [0.07123292982578278, 0.08547176420688629, -0.001888219267129898], "cos_innovation_negg": [0.323596328496933, 0.4039571285247803, 0.7828824520111084], "cos_apical_negg": [0.07123292237520218, 0.08547176420688629, -0.0018882108852267265], "cos_Ac_negg": [0.32363253831863403, 0.4039892554283142, 0.7832027673721313], "r_norm": [0.469385027885437, 0.46704259514808655, 0.4649215638637543], "ldr": {"dL_sdil": -0.022368118166923523, "dL_bp": -0.0630473643541336, "ratio": 0.35478276365817646}}, {"step": 3600, "epoch": 7, "train_loss": 0.21834729611873627, "cos_r_negg": [0.05272437632083893, 0.0676371306180954, -0.10575323551893234], "cos_innovation_negg": [0.29076385498046875, 0.3381834328174591, 0.7723140716552734], "cos_apical_negg": [0.05272437632083893, 0.0676371231675148, -0.10575324296951294], "cos_Ac_negg": [0.2907887399196625, 0.33820831775665283, 0.7725608348846436], "r_norm": [0.4339298903942108, 0.4317660331726074, 0.4260828495025635]}, {"step": 3700, "epoch": 7, "train_loss": 0.1404748260974884, "cos_r_negg": [0.08349102735519409, 0.11089466512203217, -0.02220269665122032], "cos_innovation_negg": [0.3270338773727417, 0.399289071559906, 0.834686815738678], "cos_apical_negg": [0.08349102735519409, 0.11089465767145157, -0.022202705964446068], "cos_Ac_negg": [0.326878160238266, 0.3994574546813965, 0.8365737795829773], "r_norm": [0.5048480033874512, 0.49873921275138855, 0.4960118532180786]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.952, "test_loss": 0.22041663665771485}, {"step": 3800, "epoch": 8, "train_loss": 0.13137498497962952, "cos_r_negg": [0.05077745020389557, 0.07475003600120544, -0.00721175130456686], "cos_innovation_negg": [0.34567296504974365, 0.4015766680240631, 0.7575458288192749], "cos_apical_negg": [0.05077744647860527, 0.07475004345178604, -0.0072117461822927], "cos_Ac_negg": [0.34569865465164185, 0.4016164243221283, 0.7581431865692139], "r_norm": [0.2985716462135315, 0.29656368494033813, 0.2936103940010071]}, {"step": 3900, "epoch": 8, "train_loss": 0.3350260853767395, "cos_r_negg": [0.05436886101961136, 0.053336240351200104, -0.016497645527124405], "cos_innovation_negg": [0.3389146327972412, 0.39224326610565186, 0.7746659517288208], "cos_apical_negg": [0.05436886101961136, 0.0533362478017807, -0.01649763435125351], "cos_Ac_negg": [0.3390335440635681, 0.3922542333602905, 0.776329755783081], "r_norm": [0.33099842071533203, 0.33080726861953735, 0.32877302169799805]}, {"step": 4000, "epoch": 8, "train_loss": 0.11113880574703217, "cos_r_negg": [0.06850633025169373, 0.09735101461410522, -0.06998926401138306], "cos_innovation_negg": [0.3288242816925049, 0.4119166433811188, 0.833611249923706], "cos_apical_negg": [0.06850633025169373, 0.09735101461410522, -0.06998926401138306], "cos_Ac_negg": [0.32886016368865967, 0.41191208362579346, 0.8338871002197266], "r_norm": [0.37500643730163574, 0.37148207426071167, 0.36656415462493896], "ldr": {"dL_sdil": -0.01134270429611206, "dL_bp": -0.029285520315170288, "ratio": 0.38731441934587685}}, {"step": 4100, "epoch": 8, "train_loss": 0.0858364850282669, "cos_r_negg": [0.05254334956407547, 0.08471120893955231, 0.044003650546073914], "cos_innovation_negg": [0.3330491781234741, 0.42515432834625244, 0.7540810108184814], "cos_apical_negg": [0.05254334956407547, 0.08471120148897171, 0.04400365799665451], "cos_Ac_negg": [0.3330346941947937, 0.42520666122436523, 0.7553086280822754], "r_norm": [0.31830069422721863, 0.31600093841552734, 0.313393235206604]}, {"step": 4200, "epoch": 8, "train_loss": 0.21423295140266418, "cos_r_negg": [0.052302129566669464, 0.06297570466995239, 0.0700845941901207], "cos_innovation_negg": [0.30239492654800415, 0.375344455242157, 0.7851440906524658], "cos_apical_negg": [0.05230212211608887, 0.06297571212053299, 0.0700845867395401], "cos_Ac_negg": [0.30278319120407104, 0.37535178661346436, 0.7863817811012268], "r_norm": [0.35025322437286377, 0.34856319427490234, 0.3455694019794464]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9539, "test_loss": 0.19512065200805664}, {"step": 4300, "epoch": 9, "train_loss": 0.11183387041091919, "cos_r_negg": [0.06085364148020744, 0.08281045407056808, 0.04685680568218231], "cos_innovation_negg": [0.36241286993026733, 0.4000053107738495, 0.7846133708953857], "cos_apical_negg": [0.06085364520549774, 0.08281046152114868, 0.046856798231601715], "cos_Ac_negg": [0.3621707856655121, 0.3999529182910919, 0.7876186966896057], "r_norm": [0.3191290497779846, 0.3193449378013611, 0.31352612376213074]}, {"step": 4400, "epoch": 9, "train_loss": 0.09203710407018661, "cos_r_negg": [0.010586235672235489, 0.05487773194909096, -0.001188170164823532], "cos_innovation_negg": [0.33382850885391235, 0.370858371257782, 0.7826945781707764], "cos_apical_negg": [0.010586238466203213, 0.054877735674381256, -0.0011881599202752113], "cos_Ac_negg": [0.3339824080467224, 0.3710342049598694, 0.7837940454483032], "r_norm": [0.30592358112335205, 0.3065817654132843, 0.3033285439014435]}, {"step": 4500, "epoch": 9, "train_loss": 0.14670249819755554, "cos_r_negg": [0.03256170451641083, 0.025534939020872116, 0.06166062131524086], "cos_innovation_negg": [0.308906614780426, 0.3818179965019226, 0.7649334073066711], "cos_apical_negg": [0.03256170079112053, 0.025534942746162415, 0.06166062504053116], "cos_Ac_negg": [0.30893397331237793, 0.381836473941803, 0.7650768756866455], "r_norm": [0.3142193853855133, 0.3119533061981201, 0.3101770877838135], "ldr": {"dL_sdil": -0.010576114058494568, "dL_bp": -0.028320685029029846, "ratio": 0.37344132204618724}}, {"step": 4600, "epoch": 9, "train_loss": 0.07353316992521286, "cos_r_negg": [0.04620254039764404, 0.07679638266563416, -0.11056393384933472], "cos_innovation_negg": [0.3613104224205017, 0.4161733388900757, 0.759416401386261], "cos_apical_negg": [0.04620254039764404, 0.07679639011621475, -0.11056393384933472], "cos_Ac_negg": [0.3614482879638672, 0.4162774682044983, 0.7612507343292236], "r_norm": [0.3280354142189026, 0.3250778913497925, 0.32129624485969543]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9558, "test_loss": 0.1965473617553711}, {"step": 4700, "epoch": 10, "train_loss": 0.1761542111635208, "cos_r_negg": [0.07049281895160675, 0.11861543357372284, 0.03404180333018303], "cos_innovation_negg": [0.3623608946800232, 0.4455554485321045, 0.80010986328125], "cos_apical_negg": [0.07049281895160675, 0.11861544847488403, 0.03404179960489273], "cos_Ac_negg": [0.36237210035324097, 0.44563212990760803, 0.8004482388496399], "r_norm": [0.36092445254325867, 0.36185771226882935, 0.3544063866138458]}, {"step": 4800, "epoch": 10, "train_loss": 0.1360308825969696, "cos_r_negg": [0.06646241992712021, 0.10535098612308502, 0.012459789402782917], "cos_innovation_negg": [0.37565308809280396, 0.4786906838417053, 0.8290354609489441], "cos_apical_negg": [0.06646241992712021, 0.10535098612308502, 0.012459791265428066], "cos_Ac_negg": [0.3758143484592438, 0.4787967801094055, 0.8300274014472961], "r_norm": [0.4202544689178467, 0.41712766885757446, 0.4099544584751129]}, {"step": 4900, "epoch": 10, "train_loss": 0.22283416986465454, "cos_r_negg": [0.06036195904016495, 0.05998602509498596, -0.07187195867300034], "cos_innovation_negg": [0.35083991289138794, 0.42936575412750244, 0.8062896728515625], "cos_apical_negg": [0.06036195531487465, 0.05998602136969566, -0.07187195122241974], "cos_Ac_negg": [0.3509412705898285, 0.4294760823249817, 0.8087453842163086], "r_norm": [0.3145882189273834, 0.314363956451416, 0.31073275208473206]}, {"step": 5000, "epoch": 10, "train_loss": 0.06285490095615387, "cos_r_negg": [0.060180436819791794, 0.0854584276676178, -0.022052615880966187], "cos_innovation_negg": [0.32612377405166626, 0.3969753682613373, 0.8318135738372803], "cos_apical_negg": [0.06018044054508209, 0.0854584276676178, -0.022052612155675888], "cos_Ac_negg": [0.3262039124965668, 0.3972184658050537, 0.8320704102516174], "r_norm": [0.3601338863372803, 0.3599690794944763, 0.3576597571372986], "ldr": {"dL_sdil": -0.009382858872413635, "dL_bp": -0.02074483036994934, "ratio": 0.45229865489792137}}, {"step": 5100, "epoch": 10, "train_loss": 0.2095557451248169, "cos_r_negg": [0.062476351857185364, 0.08328382670879364, 0.03801174461841583], "cos_innovation_negg": [0.33744996786117554, 0.38702964782714844, 0.810461163520813], "cos_apical_negg": [0.062476351857185364, 0.08328382670879364, 0.03801174461841583], "cos_Ac_negg": [0.3376238942146301, 0.38713186979293823, 0.8109366297721863], "r_norm": [0.33203983306884766, 0.33152520656585693, 0.32461535930633545]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9549, "test_loss": 0.21670466003417968}, {"step": 5200, "epoch": 11, "train_loss": 0.03744029998779297, "cos_r_negg": [0.054571449756622314, 0.061408575624227524, 0.007802973501384258], "cos_innovation_negg": [0.31328046321868896, 0.37786364555358887, 0.7863339185714722], "cos_apical_negg": [0.05457145720720291, 0.061408571898937225, 0.007802977692335844], "cos_Ac_negg": [0.3134727478027344, 0.37806326150894165, 0.7867177724838257], "r_norm": [0.3781473934650421, 0.3783072829246521, 0.37021905183792114]}, {"step": 5300, "epoch": 11, "train_loss": 0.08511867374181747, "cos_r_negg": [0.05521827191114426, 0.047149673104286194, -0.07350342720746994], "cos_innovation_negg": [0.319610595703125, 0.3657267391681671, 0.7311209440231323], "cos_apical_negg": [0.05521826818585396, 0.047149673104286194, -0.07350341975688934], "cos_Ac_negg": [0.31965216994285583, 0.366186261177063, 0.7322784662246704], "r_norm": [0.27844810485839844, 0.27682051062583923, 0.27256643772125244]}, {"step": 5400, "epoch": 11, "train_loss": 0.2059793919324875, "cos_r_negg": [0.029895169660449028, 0.045490097254514694, -0.017663519829511642], "cos_innovation_negg": [0.31468403339385986, 0.36690133810043335, 0.7841603755950928], "cos_apical_negg": [0.02989516407251358, 0.045490097254514694, -0.017663512378931046], "cos_Ac_negg": [0.31474757194519043, 0.3673539161682129, 0.7852257490158081], "r_norm": [0.2694922387599945, 0.2702241539955139, 0.2645063102245331]}, {"step": 5500, "epoch": 11, "train_loss": 0.09793797135353088, "cos_r_negg": [0.034121252596378326, 0.04588273912668228, -0.04683388024568558], "cos_innovation_negg": [0.31320059299468994, 0.3604232966899872, 0.8476406335830688], "cos_apical_negg": [0.034121252596378326, 0.04588274285197258, -0.04683388024568558], "cos_Ac_negg": [0.31333494186401367, 0.3606365919113159, 0.8482233285903931], "r_norm": [0.22828425467014313, 0.2294820249080658, 0.22452183067798615], "ldr": {"dL_sdil": -0.0037910640239715576, "dL_bp": -0.013266503810882568, "ratio": 0.28576210266202406}}, {"step": 5600, "epoch": 11, "train_loss": 0.10527027398347855, "cos_r_negg": [0.03166908770799637, 0.07440422475337982, 0.05370244383811951], "cos_innovation_negg": [0.35063043236732483, 0.3915204107761383, 0.8175175189971924], "cos_apical_negg": [0.03166908770799637, 0.07440422475337982, 0.05370244383811951], "cos_Ac_negg": [0.3509652614593506, 0.3916023075580597, 0.8232982754707336], "r_norm": [0.29442423582077026, 0.2978084683418274, 0.28816986083984375]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9524, "test_loss": 0.2283457244873047}, {"step": 5700, "epoch": 12, "train_loss": 0.04933105409145355, "cos_r_negg": [0.03611262887716293, 0.03309449926018715, 0.029374469071626663], "cos_innovation_negg": [0.3330172002315521, 0.36503297090530396, 0.7707983255386353], "cos_apical_negg": [0.036112625151872635, 0.03309449926018715, 0.02937447279691696], "cos_Ac_negg": [0.3331540822982788, 0.36441802978515625, 0.7707359790802002], "r_norm": [0.2852901220321655, 0.2859712839126587, 0.27818062901496887]}, {"step": 5800, "epoch": 12, "train_loss": 0.027510691434144974, "cos_r_negg": [0.03895428404211998, 0.0037397295236587524, -0.012944970279932022], "cos_innovation_negg": [0.31302791833877563, 0.3721868097782135, 0.8408054113388062], "cos_apical_negg": [0.03895428404211998, 0.0037397309206426144, -0.012944970279932022], "cos_Ac_negg": [0.3129417300224304, 0.372465044260025, 0.8413740396499634], "r_norm": [0.24204939603805542, 0.24285176396369934, 0.23730255663394928]}, {"step": 5900, "epoch": 12, "train_loss": 0.07010570168495178, "cos_r_negg": [0.045779164880514145, 0.025905540212988853, -0.03397796303033829], "cos_innovation_negg": [0.30706587433815, 0.3653751015663147, 0.8236417770385742], "cos_apical_negg": [0.045779161155223846, 0.025905543938279152, -0.03397796303033829], "cos_Ac_negg": [0.30687054991722107, 0.36525511741638184, 0.8233067393302917], "r_norm": [0.24876734614372253, 0.25016099214553833, 0.24351221323013306]}, {"step": 6000, "epoch": 12, "train_loss": 0.07453566044569016, "cos_r_negg": [0.051743581891059875, 0.06081479787826538, -0.09986754506826401], "cos_innovation_negg": [0.2523764967918396, 0.33205777406692505, 0.7742416858673096], "cos_apical_negg": [0.051743581891059875, 0.06081479415297508, -0.09986754506826401], "cos_Ac_negg": [0.25214993953704834, 0.33208534121513367, 0.7745264768600464], "r_norm": [0.3025183081626892, 0.3031054139137268, 0.2970903217792511], "ldr": {"dL_sdil": -0.014887869358062744, "dL_bp": -0.04170909523963928, "ratio": 0.35694539218663474}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9587, "test_loss": 0.19036913528442384}, {"step": 6100, "epoch": 13, "train_loss": 0.12489429116249084, "cos_r_negg": [0.01868647150695324, 0.05009150132536888, -0.05036555603146553], "cos_innovation_negg": [0.28156161308288574, 0.325544536113739, 0.8319211006164551], "cos_apical_negg": [0.01868647336959839, 0.05009150505065918, -0.05036555975675583], "cos_Ac_negg": [0.2816319763660431, 0.32549822330474854, 0.831241250038147], "r_norm": [0.21395304799079895, 0.21516923606395721, 0.20954222977161407]}, {"step": 6200, "epoch": 13, "train_loss": 0.06980135291814804, "cos_r_negg": [0.024264836683869362, 0.04623847454786301, 0.02560235559940338], "cos_innovation_negg": [0.26402226090431213, 0.29349374771118164, 0.8001523613929749], "cos_apical_negg": [0.024264834821224213, 0.04623847082257271, 0.025602353736758232], "cos_Ac_negg": [0.2640908360481262, 0.2936093211174011, 0.8022770881652832], "r_norm": [0.25522172451019287, 0.254529744386673, 0.24757462739944458]}, {"step": 6300, "epoch": 13, "train_loss": 0.0552789568901062, "cos_r_negg": [0.04411857947707176, 0.052620403468608856, -0.08288753032684326], "cos_innovation_negg": [0.284152090549469, 0.3402290940284729, 0.8187436461448669], "cos_apical_negg": [0.044118575751781464, 0.052620403468608856, -0.08288752287626266], "cos_Ac_negg": [0.28412461280822754, 0.34024399518966675, 0.8188440203666687], "r_norm": [0.2694920599460602, 0.2706873118877411, 0.26445186138153076]}, {"step": 6400, "epoch": 13, "train_loss": 0.09508123993873596, "cos_r_negg": [0.03596219792962074, 0.09026847779750824, -0.20733079314231873], "cos_innovation_negg": [0.24826094508171082, 0.31383296847343445, 0.8321732878684998], "cos_apical_negg": [0.03596220165491104, 0.09026847779750824, -0.20733079314231873], "cos_Ac_negg": [0.2486109733581543, 0.3139948844909668, 0.8320057392120361], "r_norm": [0.34740346670150757, 0.34955161809921265, 0.3384225368499756]}, {"step": 6500, "epoch": 13, "train_loss": 0.02187994308769703, "cos_r_negg": [0.02673962339758873, 0.05946764349937439, 0.0289677232503891], "cos_innovation_negg": [0.33422619104385376, 0.409740686416626, 0.789715051651001], "cos_apical_negg": [0.02673962339758873, 0.05946763604879379, 0.02896771766245365], "cos_Ac_negg": [0.33436548709869385, 0.40976014733314514, 0.7899283170700073], "r_norm": [0.28118497133255005, 0.2813083529472351, 0.2722208499908447], "ldr": {"dL_sdil": -0.012864410877227783, "dL_bp": -0.033224016427993774, "ratio": 0.387202158568298}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9583, "test_loss": 0.20578976516723632}, {"step": 6600, "epoch": 14, "train_loss": 0.015072842128574848, "cos_r_negg": [0.029648568481206894, 0.07078063488006592, 0.05156379193067551], "cos_innovation_negg": [0.3278260827064514, 0.3919757306575775, 0.8455908298492432], "cos_apical_negg": [0.029648566618561745, 0.07078063488006592, 0.05156379193067551], "cos_Ac_negg": [0.32804664969444275, 0.3920831084251404, 0.8467540740966797], "r_norm": [0.2822783887386322, 0.28691914677619934, 0.27822020649909973]}, {"step": 6700, "epoch": 14, "train_loss": 0.09784422814846039, "cos_r_negg": [0.03921347111463547, 0.07141529023647308, -0.0009423643350601196], "cos_innovation_negg": [0.289802610874176, 0.33943602442741394, 0.7788090705871582], "cos_apical_negg": [0.039213474839925766, 0.07141529768705368, -0.0009423680603504181], "cos_Ac_negg": [0.29139119386672974, 0.34089693427085876, 0.7850428819656372], "r_norm": [0.3199389576911926, 0.32207179069519043, 0.30854177474975586]}, {"step": 6800, "epoch": 14, "train_loss": 0.09813250601291656, "cos_r_negg": [0.014461950398981571, 0.06348966062068939, -0.06751073151826859], "cos_innovation_negg": [0.3077033758163452, 0.39028069376945496, 0.8379182815551758], "cos_apical_negg": [0.01446195226162672, 0.06348966807126999, -0.06751073151826859], "cos_Ac_negg": [0.3083142936229706, 0.390658974647522, 0.8378673195838928], "r_norm": [0.2994158864021301, 0.29872339963912964, 0.2900228500366211]}, {"step": 6900, "epoch": 14, "train_loss": 0.08681008964776993, "cos_r_negg": [0.033321674913167953, 0.07002883404493332, -0.0655941367149353], "cos_innovation_negg": [0.3325360119342804, 0.38789814710617065, 0.8107591867446899], "cos_apical_negg": [0.033321674913167953, 0.07002883404493332, -0.0655941367149353], "cos_Ac_negg": [0.33260440826416016, 0.38798093795776367, 0.8117262125015259], "r_norm": [0.2516554296016693, 0.2534027695655823, 0.2475077211856842]}, {"step": 7000, "epoch": 14, "train_loss": 0.059283297508955, "cos_r_negg": [0.01784803718328476, 0.06584445387125015, -0.050615355372428894], "cos_innovation_negg": [0.3058047890663147, 0.4031923711299896, 0.8342921733856201], "cos_apical_negg": [0.01784803718328476, 0.06584444642066956, -0.050615355372428894], "cos_Ac_negg": [0.30602312088012695, 0.4030851721763611, 0.8347792625427246], "r_norm": [0.25192874670028687, 0.2516576051712036, 0.24358385801315308], "ldr": {"dL_sdil": -0.006216078996658325, "dL_bp": -0.015110492706298828, "ratio": 0.4113750039445864}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9608, "test_loss": 0.18723466873168945}], "final": {"test_acc": 0.9608, "test_loss": 0.18723466873168945, "wall_s": 25.586419343948364, "cos_r_negg": [0.02722158655524254, 0.043768249452114105, -0.06765811145305634], "cos_innovation_negg": [0.32139062881469727, 0.3915737271308899, 0.7970538139343262], "cos_apical_negg": [0.027221590280532837, 0.043768249452114105, -0.06765811890363693], "cos_Ac_negg": [0.3217700123786926, 0.3921627402305603, 0.7975997924804688]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_matched_s1.json b/results/nuis_ctrl_v1_rho0p2_matched_s1.json new file mode 100644 index 0000000..da4dab3 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_matched_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_matched_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.017240727320313454, -0.012393837794661522, -0.008352074772119522], "cos_innovation_negg": [0.020772531628608704, -0.013317099772393703, -0.007092906627804041], "cos_apical_negg": [0.017240727320313454, -0.012393837794661522, -0.008352073840796947], "cos_Ac_negg": [0.02077198028564453, -0.01331748254597187, -0.007093104999512434], "r_norm": [4.737682819366455, 4.7974700927734375, 4.787670135498047], "ldr": {"dL_sdil": -0.025846004486083984, "dL_bp": -0.19701766967773438, "ratio": 0.13118622572463068}}, {"step": 100, "epoch": 0, "train_loss": 0.43666037917137146, "cos_r_negg": [0.20835015177726746, 0.25481268763542175, 0.43818509578704834], "cos_innovation_negg": [0.38202139735221863, 0.4236884117126465, 0.7178184986114502], "cos_apical_negg": [0.20835013687610626, 0.25481265783309937, 0.43818509578704834], "cos_Ac_negg": [0.38195955753326416, 0.42365947365760803, 0.7178199887275696], "r_norm": [1.5006263256072998, 1.4957764148712158, 1.5314254760742188]}, {"step": 200, "epoch": 0, "train_loss": 0.5769292116165161, "cos_r_negg": [0.17596173286437988, 0.24774552881717682, 0.3612061142921448], "cos_innovation_negg": [0.37133896350860596, 0.41942042112350464, 0.7078418135643005], "cos_apical_negg": [0.17596173286437988, 0.24774552881717682, 0.3612061142921448], "cos_Ac_negg": [0.3711877465248108, 0.4193950593471527, 0.7078851461410522], "r_norm": [1.2929306030273438, 1.298997402191162, 1.3352806568145752]}, {"step": 300, "epoch": 0, "train_loss": 0.41536980867385864, "cos_r_negg": [0.16681239008903503, 0.19450592994689941, 0.15890663862228394], "cos_innovation_negg": [0.38078343868255615, 0.4407312870025635, 0.7362583875656128], "cos_apical_negg": [0.16681239008903503, 0.19450592994689941, 0.15890663862228394], "cos_Ac_negg": [0.3807200491428375, 0.44068896770477295, 0.7362213134765625], "r_norm": [1.0151933431625366, 1.0129313468933105, 1.0302326679229736]}, {"step": 400, "epoch": 0, "train_loss": 0.3051149249076843, "cos_r_negg": [0.1531429886817932, 0.1752968281507492, 0.34554895758628845], "cos_innovation_negg": [0.4214603304862976, 0.4753772020339966, 0.7752066850662231], "cos_apical_negg": [0.1531429886817932, 0.175296813249588, 0.34554895758628845], "cos_Ac_negg": [0.4213849604129791, 0.4753345847129822, 0.7751826047897339], "r_norm": [0.8816912174224854, 0.8831709027290344, 0.896199107170105]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9204, "test_loss": 0.3181631790161133}, {"step": 500, "epoch": 1, "train_loss": 0.31391698122024536, "cos_r_negg": [0.12520919740200043, 0.1143871396780014, 0.35029834508895874], "cos_innovation_negg": [0.39721906185150146, 0.4291078448295593, 0.7182943224906921], "cos_apical_negg": [0.12520919740200043, 0.1143871396780014, 0.35029837489128113], "cos_Ac_negg": [0.3971734344959259, 0.42905813455581665, 0.7181919813156128], "r_norm": [0.7292766571044922, 0.7327057123184204, 0.7424994111061096], "ldr": {"dL_sdil": -0.008875042200088501, "dL_bp": -0.013275712728500366, "ratio": 0.6685171923790966}}, {"step": 600, "epoch": 1, "train_loss": 0.19112785160541534, "cos_r_negg": [0.14094552397727966, 0.158867746591568, 0.37589722871780396], "cos_innovation_negg": [0.3731793761253357, 0.41393327713012695, 0.7278618812561035], "cos_apical_negg": [0.14094550907611847, 0.158867746591568, 0.37589719891548157], "cos_Ac_negg": [0.3729149103164673, 0.41383838653564453, 0.7277028560638428], "r_norm": [0.8692857027053833, 0.8650484085083008, 0.8904972672462463]}, {"step": 700, "epoch": 1, "train_loss": 0.3435584306716919, "cos_r_negg": [0.17268341779708862, 0.1817837804555893, 0.37230372428894043], "cos_innovation_negg": [0.3900091052055359, 0.4602586328983307, 0.71354740858078], "cos_apical_negg": [0.17268341779708862, 0.1817837804555893, 0.37230372428894043], "cos_Ac_negg": [0.38987860083580017, 0.4600508213043213, 0.7132584452629089], "r_norm": [0.9967005252838135, 1.00111985206604, 1.025026798248291]}, {"step": 800, "epoch": 1, "train_loss": 0.27373164892196655, "cos_r_negg": [0.12529520690441132, 0.13700753450393677, 0.3232805132865906], "cos_innovation_negg": [0.36092323064804077, 0.42001402378082275, 0.6527142524719238], "cos_apical_negg": [0.12529519200325012, 0.13700753450393677, 0.3232804834842682], "cos_Ac_negg": [0.36077916622161865, 0.41987794637680054, 0.6526259183883667], "r_norm": [0.8122239708900452, 0.8109956979751587, 0.8370965123176575]}, {"step": 900, "epoch": 1, "train_loss": 0.3742179870605469, "cos_r_negg": [0.11092528700828552, 0.12023036181926727, 0.3318372070789337], "cos_innovation_negg": [0.34369343519210815, 0.451210081577301, 0.7852743864059448], "cos_apical_negg": [0.11092528700828552, 0.12023036926984787, 0.3318372368812561], "cos_Ac_negg": [0.34344062209129333, 0.4509604871273041, 0.7851155996322632], "r_norm": [0.6117479801177979, 0.6097850799560547, 0.6194773316383362]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9256, "test_loss": 0.2996609817504883}, {"step": 1000, "epoch": 2, "train_loss": 0.1929795891046524, "cos_r_negg": [0.1302143633365631, 0.14919152855873108, 0.3366738557815552], "cos_innovation_negg": [0.3433101773262024, 0.4554767608642578, 0.7438793182373047], "cos_apical_negg": [0.1302143633365631, 0.14919152855873108, 0.33667388558387756], "cos_Ac_negg": [0.3433893918991089, 0.4553982615470886, 0.7432876229286194], "r_norm": [0.7970943450927734, 0.7988111972808838, 0.8184425830841064], "ldr": {"dL_sdil": -0.05394434928894043, "dL_bp": -0.0796496719121933, "ratio": 0.6772702007914018}}, {"step": 1100, "epoch": 2, "train_loss": 0.3101557493209839, "cos_r_negg": [0.10145506262779236, 0.14104238152503967, 0.19794082641601562], "cos_innovation_negg": [0.3433656096458435, 0.4354109764099121, 0.6746965646743774], "cos_apical_negg": [0.10145506262779236, 0.14104238152503967, 0.19794082641601562], "cos_Ac_negg": [0.3433460295200348, 0.43531334400177, 0.6745729446411133], "r_norm": [0.7037591338157654, 0.7010056972503662, 0.7048548460006714]}, {"step": 1200, "epoch": 2, "train_loss": 0.3756130039691925, "cos_r_negg": [0.1217937171459198, 0.15203003585338593, 0.16101674735546112], "cos_innovation_negg": [0.35294920206069946, 0.44110107421875, 0.7587949633598328], "cos_apical_negg": [0.1217937171459198, 0.15203005075454712, 0.16101673245429993], "cos_Ac_negg": [0.35286062955856323, 0.44106727838516235, 0.7587511539459229], "r_norm": [0.751081109046936, 0.7524017691612244, 0.7581551671028137]}, {"step": 1300, "epoch": 2, "train_loss": 0.2943981885910034, "cos_r_negg": [0.12460248917341232, 0.15353891253471375, 0.2434411346912384], "cos_innovation_negg": [0.3823387622833252, 0.4978301525115967, 0.7680021524429321], "cos_apical_negg": [0.12460248917341232, 0.15353889763355255, 0.2434411346912384], "cos_Ac_negg": [0.38236328959465027, 0.4978780150413513, 0.7679839730262756], "r_norm": [0.7431914210319519, 0.7390936613082886, 0.7455476522445679]}, {"step": 1400, "epoch": 2, "train_loss": 0.30064770579338074, "cos_r_negg": [0.10108630359172821, 0.12271721661090851, 0.10179692506790161], "cos_innovation_negg": [0.3069431185722351, 0.41232383251190186, 0.7208117842674255], "cos_apical_negg": [0.10108629614114761, 0.12271721661090851, 0.10179691016674042], "cos_Ac_negg": [0.30691492557525635, 0.41230303049087524, 0.7208303213119507], "r_norm": [0.6726988554000854, 0.6703563928604126, 0.6752356290817261]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9427, "test_loss": 0.2344528190612793}, {"step": 1500, "epoch": 3, "train_loss": 0.2498413324356079, "cos_r_negg": [0.13519391417503357, 0.13965395092964172, 0.24174046516418457], "cos_innovation_negg": [0.3113863170146942, 0.35737884044647217, 0.6559958457946777], "cos_apical_negg": [0.13519391417503357, 0.13965395092964172, 0.24174046516418457], "cos_Ac_negg": [0.3113986551761627, 0.3573784828186035, 0.6557502746582031], "r_norm": [0.7976827621459961, 0.7960426807403564, 0.8066877126693726], "ldr": {"dL_sdil": -0.0325922966003418, "dL_bp": -0.05117088556289673, "ratio": 0.6369304779820735}}, {"step": 1600, "epoch": 3, "train_loss": 0.2467401921749115, "cos_r_negg": [0.10957890003919601, 0.0609438382089138, 0.058589447289705276], "cos_innovation_negg": [0.3661121129989624, 0.43212515115737915, 0.6990386247634888], "cos_apical_negg": [0.10957890003919601, 0.060943834483623505, 0.058589451014995575], "cos_Ac_negg": [0.3661012053489685, 0.43211573362350464, 0.6989883184432983], "r_norm": [0.6198312640190125, 0.6218320727348328, 0.6282820105552673]}, {"step": 1700, "epoch": 3, "train_loss": 0.20970016717910767, "cos_r_negg": [0.10024343430995941, 0.07061778008937836, 0.08824867010116577], "cos_innovation_negg": [0.30672937631607056, 0.44945603609085083, 0.7075759172439575], "cos_apical_negg": [0.10024343430995941, 0.07061778008937836, 0.08824867010116577], "cos_Ac_negg": [0.30671191215515137, 0.4494774341583252, 0.7076292037963867], "r_norm": [0.5842924118041992, 0.5908018350601196, 0.593904972076416]}, {"step": 1800, "epoch": 3, "train_loss": 0.0652875304222107, "cos_r_negg": [0.10777616500854492, 0.11677993834018707, 0.19906827807426453], "cos_innovation_negg": [0.3133244812488556, 0.38910147547721863, 0.6052562594413757], "cos_apical_negg": [0.10777615755796432, 0.11677993834018707, 0.19906826317310333], "cos_Ac_negg": [0.3133697211742401, 0.3892460763454437, 0.6052983403205872], "r_norm": [0.6780498623847961, 0.6789780259132385, 0.6806283593177795]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9394, "test_loss": 0.23556280670166016}, {"step": 1900, "epoch": 4, "train_loss": 0.20867076516151428, "cos_r_negg": [0.09630155563354492, 0.09928036481142044, 0.08963586390018463], "cos_innovation_negg": [0.3106575608253479, 0.3883098363876343, 0.6949050426483154], "cos_apical_negg": [0.09630155563354492, 0.09928037226200104, 0.08963586390018463], "cos_Ac_negg": [0.31062984466552734, 0.3882838785648346, 0.6951161623001099], "r_norm": [0.5608504414558411, 0.5664485096931458, 0.5688900947570801]}, {"step": 2000, "epoch": 4, "train_loss": 0.2140769213438034, "cos_r_negg": [0.09616347402334213, 0.09905596077442169, 0.05964657664299011], "cos_innovation_negg": [0.3518545627593994, 0.4386337697505951, 0.7240620851516724], "cos_apical_negg": [0.09616347402334213, 0.09905596077442169, 0.059646572917699814], "cos_Ac_negg": [0.3518730103969574, 0.43865907192230225, 0.72425377368927], "r_norm": [0.5334855318069458, 0.5381511449813843, 0.5414948463439941], "ldr": {"dL_sdil": -0.026597067713737488, "dL_bp": -0.04154585301876068, "ratio": 0.6401858616725757}}, {"step": 2100, "epoch": 4, "train_loss": 0.19390198588371277, "cos_r_negg": [0.09013150632381439, 0.10562118887901306, 0.07240511476993561], "cos_innovation_negg": [0.2892338037490845, 0.38961562514305115, 0.5738331079483032], "cos_apical_negg": [0.09013150632381439, 0.10562118887901306, 0.0724051222205162], "cos_Ac_negg": [0.28925561904907227, 0.3895808160305023, 0.5740745067596436], "r_norm": [0.506035566329956, 0.5089210867881775, 0.5081740021705627]}, {"step": 2200, "epoch": 4, "train_loss": 0.11691835522651672, "cos_r_negg": [0.08761823922395706, 0.09638509154319763, 0.059242382645606995], "cos_innovation_negg": [0.32944047451019287, 0.4330959916114807, 0.7013381719589233], "cos_apical_negg": [0.08761824667453766, 0.09638509154319763, 0.05924238637089729], "cos_Ac_negg": [0.32943394780158997, 0.433062881231308, 0.7013884782791138], "r_norm": [0.506537914276123, 0.5124306678771973, 0.5145182609558105]}, {"step": 2300, "epoch": 4, "train_loss": 0.32473093271255493, "cos_r_negg": [0.06946565955877304, 0.1047610491514206, 0.12642177939414978], "cos_innovation_negg": [0.37157824635505676, 0.47215232253074646, 0.6800374984741211], "cos_apical_negg": [0.06946565955877304, 0.1047610491514206, 0.12642177939414978], "cos_Ac_negg": [0.37152987718582153, 0.4721730947494507, 0.6800609827041626], "r_norm": [0.5161682367324829, 0.5198871493339539, 0.5282779932022095]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9467, "test_loss": 0.23900887680053712}, {"step": 2400, "epoch": 5, "train_loss": 0.2654557526111603, "cos_r_negg": [0.0997617095708847, 0.10604047775268555, 0.10383321344852448], "cos_innovation_negg": [0.3281242251396179, 0.43690720200538635, 0.6567567586898804], "cos_apical_negg": [0.0997617095708847, 0.10604047030210495, 0.10383322089910507], "cos_Ac_negg": [0.32796385884284973, 0.43690454959869385, 0.6571217775344849], "r_norm": [0.6002587080001831, 0.608529806137085, 0.6144667267799377]}, {"step": 2500, "epoch": 5, "train_loss": 0.21987125277519226, "cos_r_negg": [0.08588224649429321, 0.09207742661237717, 0.05829627439379692], "cos_innovation_negg": [0.35877493023872375, 0.45356300473213196, 0.6823073625564575], "cos_apical_negg": [0.08588224649429321, 0.09207742661237717, 0.05829627439379692], "cos_Ac_negg": [0.3587193787097931, 0.45349550247192383, 0.6824527382850647], "r_norm": [0.4181100130081177, 0.4237861931324005, 0.422446608543396], "ldr": {"dL_sdil": -0.009933114051818848, "dL_bp": -0.02018396556377411, "ratio": 0.49212896348013285}}, {"step": 2600, "epoch": 5, "train_loss": 0.14134864509105682, "cos_r_negg": [0.08585883677005768, 0.07219374179840088, -0.0022207647562026978], "cos_innovation_negg": [0.3640020787715912, 0.4138384759426117, 0.5791534185409546], "cos_apical_negg": [0.08585882186889648, 0.07219374179840088, -0.0022207628935575485], "cos_Ac_negg": [0.36405181884765625, 0.41394901275634766, 0.5790916681289673], "r_norm": [0.4071858525276184, 0.4091227352619171, 0.41021162271499634]}, {"step": 2700, "epoch": 5, "train_loss": 0.1647357940673828, "cos_r_negg": [0.08022316545248032, 0.1015097051858902, 0.1727856695652008], "cos_innovation_negg": [0.3466169834136963, 0.4389486014842987, 0.6225054860115051], "cos_apical_negg": [0.08022316545248032, 0.1015097051858902, 0.1727856546640396], "cos_Ac_negg": [0.34710901975631714, 0.4392862915992737, 0.6224606037139893], "r_norm": [0.4196070730686188, 0.4207836389541626, 0.42698976397514343]}, {"step": 2800, "epoch": 5, "train_loss": 0.3329737186431885, "cos_r_negg": [0.062061846256256104, 0.06446561962366104, -0.1068054735660553], "cos_innovation_negg": [0.31065529584884644, 0.3601764142513275, 0.6510758996009827], "cos_apical_negg": [0.062061846256256104, 0.06446562707424164, -0.1068054810166359], "cos_Ac_negg": [0.31072014570236206, 0.36022865772247314, 0.6514415740966797], "r_norm": [0.4171377420425415, 0.41613373160362244, 0.4133930206298828]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9478, "test_loss": 0.22174640655517577}, {"step": 2900, "epoch": 6, "train_loss": 0.06915221363306046, "cos_r_negg": [0.06541688740253448, 0.06289783120155334, 0.021152105182409286], "cos_innovation_negg": [0.3067023754119873, 0.3437613844871521, 0.601391077041626], "cos_apical_negg": [0.06541688740253448, 0.06289783120155334, 0.021152108907699585], "cos_Ac_negg": [0.3066386878490448, 0.3438071012496948, 0.6019645929336548], "r_norm": [0.342886745929718, 0.34700173139572144, 0.348785400390625]}, {"step": 3000, "epoch": 6, "train_loss": 0.3120661973953247, "cos_r_negg": [0.04496881738305092, 0.06896763294935226, 0.07797306776046753], "cos_innovation_negg": [0.3395608067512512, 0.4020515978336334, 0.6359972357749939], "cos_apical_negg": [0.04496881365776062, 0.06896763294935226, 0.07797306030988693], "cos_Ac_negg": [0.3396388292312622, 0.402157187461853, 0.6361735463142395], "r_norm": [0.3462325930595398, 0.34839412569999695, 0.3485502004623413], "ldr": {"dL_sdil": -0.0062240660190582275, "dL_bp": -0.018158212304115295, "ratio": 0.34276865557121133}}, {"step": 3100, "epoch": 6, "train_loss": 0.2541540265083313, "cos_r_negg": [0.051937032490968704, 0.05824144184589386, 0.05228520184755325], "cos_innovation_negg": [0.33272498846054077, 0.40439164638519287, 0.5892277359962463], "cos_apical_negg": [0.051937032490968704, 0.05824144184589386, 0.05228520184755325], "cos_Ac_negg": [0.33294934034347534, 0.4046574831008911, 0.5879822969436646], "r_norm": [0.4057489037513733, 0.4098128080368042, 0.41192352771759033]}, {"step": 3200, "epoch": 6, "train_loss": 0.1853654384613037, "cos_r_negg": [0.045160599052906036, 0.038169652223587036, -0.07041910290718079], "cos_innovation_negg": [0.2956198453903198, 0.3735000789165497, 0.6555889248847961], "cos_apical_negg": [0.045160599052906036, 0.03816964477300644, -0.07041910290718079], "cos_Ac_negg": [0.29573768377304077, 0.37365221977233887, 0.6555120944976807], "r_norm": [0.40401849150657654, 0.40795254707336426, 0.40428486466407776]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9536, "test_loss": 0.20005987472534179}, {"step": 3300, "epoch": 7, "train_loss": 0.15534937381744385, "cos_r_negg": [0.03811334818601608, 0.07582096010446548, 0.07453857362270355], "cos_innovation_negg": [0.28581511974334717, 0.38360756635665894, 0.6406189799308777], "cos_apical_negg": [0.03811334818601608, 0.07582096010446548, 0.07453857362270355], "cos_Ac_negg": [0.2858169674873352, 0.38363659381866455, 0.6405774354934692], "r_norm": [0.41912922263145447, 0.4209652543067932, 0.41943109035491943]}, {"step": 3400, "epoch": 7, "train_loss": 0.20773613452911377, "cos_r_negg": [0.06513060629367828, 0.060187600553035736, -0.11100343614816666], "cos_innovation_negg": [0.30233001708984375, 0.3845340609550476, 0.7095423340797424], "cos_apical_negg": [0.06513060629367828, 0.06018759310245514, -0.11100342869758606], "cos_Ac_negg": [0.3023809492588043, 0.38473501801490784, 0.7098117470741272], "r_norm": [0.4422343671321869, 0.445319801568985, 0.444426953792572]}, {"step": 3500, "epoch": 7, "train_loss": 0.15677744150161743, "cos_r_negg": [0.043933749198913574, 0.031805139034986496, 0.06841997057199478], "cos_innovation_negg": [0.34367355704307556, 0.43468499183654785, 0.5411227941513062], "cos_apical_negg": [0.043933745473623276, 0.031805142760276794, 0.06841996312141418], "cos_Ac_negg": [0.34403160214424133, 0.43504440784454346, 0.5417006015777588], "r_norm": [0.4290872812271118, 0.4381307363510132, 0.43283799290657043], "ldr": {"dL_sdil": -0.03834246098995209, "dL_bp": -0.08114735782146454, "ratio": 0.4725041211361537}}, {"step": 3600, "epoch": 7, "train_loss": 0.07466712594032288, "cos_r_negg": [0.06115208938717842, 0.051563311368227005, 0.026588505133986473], "cos_innovation_negg": [0.3052281141281128, 0.3607001006603241, 0.6314609050750732], "cos_apical_negg": [0.06115208938717842, 0.05156330764293671, 0.026588518172502518], "cos_Ac_negg": [0.30525103211402893, 0.3607829511165619, 0.6315804719924927], "r_norm": [0.4006863832473755, 0.40455734729766846, 0.40392962098121643]}, {"step": 3700, "epoch": 7, "train_loss": 0.041909635066986084, "cos_r_negg": [0.06996788829565048, 0.06286899000406265, 0.08115151524543762], "cos_innovation_negg": [0.3624466061592102, 0.4368343651294708, 0.6516638994216919], "cos_apical_negg": [0.06996788829565048, 0.06286898255348206, 0.08115152269601822], "cos_Ac_negg": [0.36251547932624817, 0.43685734272003174, 0.6515238285064697], "r_norm": [0.38318565487861633, 0.38588234782218933, 0.38669681549072266]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9539, "test_loss": 0.22187461624145507}, {"step": 3800, "epoch": 8, "train_loss": 0.08812721073627472, "cos_r_negg": [0.07853343337774277, 0.04422430321574211, -0.002246011048555374], "cos_innovation_negg": [0.3396430015563965, 0.3951643109321594, 0.5624436736106873], "cos_apical_negg": [0.07853343337774277, 0.04422430694103241, -0.0022460101172327995], "cos_Ac_negg": [0.3397493362426758, 0.39545124769210815, 0.563054084777832], "r_norm": [0.3282546103000641, 0.3316921293735504, 0.3330567181110382]}, {"step": 3900, "epoch": 8, "train_loss": 0.2891842722892761, "cos_r_negg": [0.05308692157268524, 0.06476743519306183, 0.06887641549110413], "cos_innovation_negg": [0.31262266635894775, 0.362143337726593, 0.591728687286377], "cos_apical_negg": [0.05308692157268524, 0.06476742774248123, 0.06887641549110413], "cos_Ac_negg": [0.31266963481903076, 0.36224740743637085, 0.592028021812439], "r_norm": [0.3937183916568756, 0.39886635541915894, 0.4006030559539795]}, {"step": 4000, "epoch": 8, "train_loss": 0.20035336911678314, "cos_r_negg": [0.031525179743766785, 0.0664583370089531, -0.025999654084444046], "cos_innovation_negg": [0.3442477881908417, 0.42902272939682007, 0.6642171144485474], "cos_apical_negg": [0.031525179743766785, 0.0664583370089531, -0.025999655947089195], "cos_Ac_negg": [0.34442564845085144, 0.42895904183387756, 0.6628882884979248], "r_norm": [0.36573290824890137, 0.3673510253429413, 0.36741286516189575], "ldr": {"dL_sdil": -0.007744699716567993, "dL_bp": -0.019544988870620728, "ratio": 0.3962498913575503}}, {"step": 4100, "epoch": 8, "train_loss": 0.06988651305437088, "cos_r_negg": [0.03828493505716324, 0.0416024774312973, -0.027418073266744614], "cos_innovation_negg": [0.3374292850494385, 0.4283182919025421, 0.6232483386993408], "cos_apical_negg": [0.03828493505716324, 0.0416024811565876, -0.027418067678809166], "cos_Ac_negg": [0.3375769555568695, 0.4284735918045044, 0.623253345489502], "r_norm": [0.3242313861846924, 0.3279925584793091, 0.32382041215896606]}, {"step": 4200, "epoch": 8, "train_loss": 0.28709572553634644, "cos_r_negg": [0.03398432210087776, 0.027570728212594986, -0.09911291301250458], "cos_innovation_negg": [0.2615605890750885, 0.35825857520103455, 0.6641868352890015], "cos_apical_negg": [0.03398432582616806, 0.027570728212594986, -0.09911291301250458], "cos_Ac_negg": [0.2626052498817444, 0.3597257733345032, 0.6723030805587769], "r_norm": [0.41140487790107727, 0.41126570105552673, 0.413004070520401]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9517, "test_loss": 0.22888046035766602}, {"step": 4300, "epoch": 9, "train_loss": 0.1808217167854309, "cos_r_negg": [0.06602910161018372, 0.06715333461761475, 0.10144918411970139], "cos_innovation_negg": [0.33686959743499756, 0.4302141070365906, 0.6566715240478516], "cos_apical_negg": [0.06602910161018372, 0.06715333461761475, 0.10144919157028198], "cos_Ac_negg": [0.3374009132385254, 0.4308498203754425, 0.6591722369194031], "r_norm": [0.36052775382995605, 0.361494243144989, 0.3588387072086334]}, {"step": 4400, "epoch": 9, "train_loss": 0.07972505688667297, "cos_r_negg": [0.0679706558585167, 0.02000126615166664, -0.06801160424947739], "cos_innovation_negg": [0.27695542573928833, 0.3563426434993744, 0.5894439220428467], "cos_apical_negg": [0.0679706484079361, 0.02000126987695694, -0.06801160424947739], "cos_Ac_negg": [0.2770719528198242, 0.3565279245376587, 0.5897455811500549], "r_norm": [0.34941565990448, 0.3531954288482666, 0.35092806816101074]}, {"step": 4500, "epoch": 9, "train_loss": 0.1966162472963333, "cos_r_negg": [0.0590222105383873, 0.048345714807510376, -0.10357405245304108], "cos_innovation_negg": [0.2976261377334595, 0.3906477093696594, 0.6366250514984131], "cos_apical_negg": [0.0590222142636776, 0.048345718532800674, -0.10357404500246048], "cos_Ac_negg": [0.2978004813194275, 0.3910141587257385, 0.6378175020217896], "r_norm": [0.37862372398376465, 0.38403183221817017, 0.38257265090942383], "ldr": {"dL_sdil": -0.022157758474349976, "dL_bp": -0.037183552980422974, "ratio": 0.5959021314078288}}, {"step": 4600, "epoch": 9, "train_loss": 0.022756293416023254, "cos_r_negg": [0.06016022711992264, 0.03513793647289276, -0.13398033380508423], "cos_innovation_negg": [0.27261465787887573, 0.3467797636985779, 0.6956620216369629], "cos_apical_negg": [0.06016022711992264, 0.03513793274760246, -0.13398033380508423], "cos_Ac_negg": [0.272876501083374, 0.34703508019447327, 0.6959861516952515], "r_norm": [0.3385137915611267, 0.3439512252807617, 0.3360411524772644]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9561, "test_loss": 0.2061817527770996}, {"step": 4700, "epoch": 10, "train_loss": 0.1913730800151825, "cos_r_negg": [0.044512998312711716, 0.03614046052098274, -0.13629215955734253], "cos_innovation_negg": [0.2882385849952698, 0.35471075773239136, 0.7138314843177795], "cos_apical_negg": [0.04451299458742142, 0.03614046424627304, -0.13629215955734253], "cos_Ac_negg": [0.28838247060775757, 0.3548765182495117, 0.7142137289047241], "r_norm": [0.30230194330215454, 0.3057623505592346, 0.30458155274391174]}, {"step": 4800, "epoch": 10, "train_loss": 0.1881817728281021, "cos_r_negg": [0.06759335845708847, 0.03726032376289368, -0.002488374710083008], "cos_innovation_negg": [0.31377869844436646, 0.3853495121002197, 0.629889965057373], "cos_apical_negg": [0.06759335845708847, 0.037260331213474274, -0.0024883784353733063], "cos_Ac_negg": [0.31391191482543945, 0.38561534881591797, 0.6300445795059204], "r_norm": [0.31095725297927856, 0.31190887093544006, 0.3091069459915161]}, {"step": 4900, "epoch": 10, "train_loss": 0.0944644957780838, "cos_r_negg": [0.08395251631736755, 0.06490360200405121, 0.1834726631641388], "cos_innovation_negg": [0.3673471510410309, 0.4356116056442261, 0.6660653948783875], "cos_apical_negg": [0.08395251631736755, 0.06490360200405121, 0.1834726631641388], "cos_Ac_negg": [0.3673630356788635, 0.43622201681137085, 0.6660963296890259], "r_norm": [0.4049217700958252, 0.40700265765190125, 0.41289329528808594]}, {"step": 5000, "epoch": 10, "train_loss": 0.024065494537353516, "cos_r_negg": [0.06475402414798737, 0.03593529760837555, -0.013042952865362167], "cos_innovation_negg": [0.3383983373641968, 0.40875038504600525, 0.71702641248703], "cos_apical_negg": [0.06475402414798737, 0.03593529015779495, -0.013042948208749294], "cos_Ac_negg": [0.3387422263622284, 0.4089089035987854, 0.7163650989532471], "r_norm": [0.3686790466308594, 0.3724566698074341, 0.36577165126800537], "ldr": {"dL_sdil": -0.01190902292728424, "dL_bp": -0.02919386327266693, "ratio": 0.4079289820622744}}, {"step": 5100, "epoch": 10, "train_loss": 0.0887155681848526, "cos_r_negg": [0.07348175346851349, 0.05021180212497711, -0.08241405338048935], "cos_innovation_negg": [0.32937365770339966, 0.3970889449119568, 0.6641775369644165], "cos_apical_negg": [0.07348175346851349, 0.05021180585026741, -0.08241405338048935], "cos_Ac_negg": [0.3292635679244995, 0.3973221778869629, 0.6646224856376648], "r_norm": [0.3370145261287689, 0.34336763620376587, 0.3394201993942261]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9531, "test_loss": 0.21261507873535157}, {"step": 5200, "epoch": 11, "train_loss": 0.015521238557994366, "cos_r_negg": [0.04192403331398964, 0.047619424760341644, -0.13480988144874573], "cos_innovation_negg": [0.29843252897262573, 0.3676559329032898, 0.6901150345802307], "cos_apical_negg": [0.04192402958869934, 0.047619421035051346, -0.13480988144874573], "cos_Ac_negg": [0.2985725998878479, 0.3679630160331726, 0.6905062198638916], "r_norm": [0.34247463941574097, 0.3466784358024597, 0.3350602388381958]}, {"step": 5300, "epoch": 11, "train_loss": 0.08481009304523468, "cos_r_negg": [0.059139396995306015, 0.10118880867958069, -0.09266543388366699], "cos_innovation_negg": [0.32587409019470215, 0.38117778301239014, 0.7519956827163696], "cos_apical_negg": [0.059139400720596313, 0.10118880867958069, -0.09266544878482819], "cos_Ac_negg": [0.3258609473705292, 0.3812134265899658, 0.7523297071456909], "r_norm": [0.3832690119743347, 0.3867148458957672, 0.38077399134635925]}, {"step": 5400, "epoch": 11, "train_loss": 0.16110804677009583, "cos_r_negg": [0.04936950281262398, 0.08688933402299881, -0.20913437008857727], "cos_innovation_negg": [0.29607468843460083, 0.37785810232162476, 0.7648166418075562], "cos_apical_negg": [0.04936949908733368, 0.08688933402299881, -0.20913435518741608], "cos_Ac_negg": [0.2964397966861725, 0.37814801931381226, 0.7652274370193481], "r_norm": [0.3694998025894165, 0.37496981024742126, 0.3623947501182556]}, {"step": 5500, "epoch": 11, "train_loss": 0.27962130308151245, "cos_r_negg": [0.05374613776803017, 0.05903695896267891, -0.08377103507518768], "cos_innovation_negg": [0.30551958084106445, 0.39926934242248535, 0.731338381767273], "cos_apical_negg": [0.053746141493320465, 0.05903695523738861, -0.08377103507518768], "cos_Ac_negg": [0.30594539642333984, 0.39952701330184937, 0.7319685816764832], "r_norm": [0.3456263244152069, 0.3510850965976715, 0.3437492847442627], "ldr": {"dL_sdil": -0.013040304183959961, "dL_bp": -0.031758204102516174, "ratio": 0.41061214109795297}}, {"step": 5600, "epoch": 11, "train_loss": 0.09887810796499252, "cos_r_negg": [0.06392280012369156, 0.06886495649814606, -0.13522015511989594], "cos_innovation_negg": [0.307576984167099, 0.3457643985748291, 0.6648601293563843], "cos_apical_negg": [0.06392280757427216, 0.06886495649814606, -0.13522017002105713], "cos_Ac_negg": [0.3086901903152466, 0.3463948667049408, 0.6666679382324219], "r_norm": [0.32832059264183044, 0.33228838443756104, 0.32474082708358765]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9564, "test_loss": 0.2051394729614258}, {"step": 5700, "epoch": 12, "train_loss": 0.06471480429172516, "cos_r_negg": [0.05594306066632271, 0.03522571921348572, -0.1324022263288498], "cos_innovation_negg": [0.34709763526916504, 0.419447124004364, 0.6934719681739807], "cos_apical_negg": [0.05594305694103241, 0.03522571921348572, -0.1324022263288498], "cos_Ac_negg": [0.34778156876564026, 0.41968727111816406, 0.6947780847549438], "r_norm": [0.3208765685558319, 0.3245876133441925, 0.3163379430770874]}, {"step": 5800, "epoch": 12, "train_loss": 0.026193108409643173, "cos_r_negg": [0.05743078514933586, 0.0580311045050621, -0.006859535351395607], "cos_innovation_negg": [0.35732418298721313, 0.3890446126461029, 0.7163714170455933], "cos_apical_negg": [0.05743078142404556, 0.0580311045050621, -0.006859548389911652], "cos_Ac_negg": [0.35760441422462463, 0.3891891837120056, 0.7167977690696716], "r_norm": [0.3208925426006317, 0.32283368706703186, 0.3142455220222473]}, {"step": 5900, "epoch": 12, "train_loss": 0.08609644323587418, "cos_r_negg": [0.03837002068758011, 0.038067497313022614, -0.10395322740077972], "cos_innovation_negg": [0.33246707916259766, 0.3791056275367737, 0.647708535194397], "cos_apical_negg": [0.03837001323699951, 0.038067497313022614, -0.10395322740077972], "cos_Ac_negg": [0.3330087959766388, 0.3796733617782593, 0.6488566398620605], "r_norm": [0.28697890043258667, 0.2914462685585022, 0.28361207246780396]}, {"step": 6000, "epoch": 12, "train_loss": 0.011555884033441544, "cos_r_negg": [0.04002008214592934, 0.022243496030569077, -0.15412598848342896], "cos_innovation_negg": [0.28803521394729614, 0.3072802424430847, 0.664710283279419], "cos_apical_negg": [0.04002008214592934, 0.022243494167923927, -0.15412598848342896], "cos_Ac_negg": [0.2880746126174927, 0.30747881531715393, 0.6650397181510925], "r_norm": [0.3107988238334656, 0.31282874941825867, 0.3014359176158905], "ldr": {"dL_sdil": -0.01149497926235199, "dL_bp": -0.028254345059394836, "ratio": 0.4068393458842466}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.955, "test_loss": 0.22476060485839844}, {"step": 6100, "epoch": 13, "train_loss": 0.019013768061995506, "cos_r_negg": [0.01851665787398815, 0.05332653224468231, -0.022532230243086815], "cos_innovation_negg": [0.35613998770713806, 0.414058655500412, 0.6762146949768066], "cos_apical_negg": [0.01851666159927845, 0.05332653596997261, -0.022532224655151367], "cos_Ac_negg": [0.3564178943634033, 0.41423872113227844, 0.6762027144432068], "r_norm": [0.25446617603302, 0.25901520252227783, 0.2508872151374817]}, {"step": 6200, "epoch": 13, "train_loss": 0.046718861907720566, "cos_r_negg": [0.03951304033398628, 0.04525625705718994, -0.09203031659126282], "cos_innovation_negg": [0.2852541208267212, 0.3206644356250763, 0.6953349113464355], "cos_apical_negg": [0.03951304405927658, 0.04525625705718994, -0.09203031659126282], "cos_Ac_negg": [0.28577062487602234, 0.32072341442108154, 0.69330894947052], "r_norm": [0.2938562035560608, 0.2982935905456543, 0.29395222663879395]}, {"step": 6300, "epoch": 13, "train_loss": 0.08005551248788834, "cos_r_negg": [0.04350179433822632, 0.04822395741939545, -0.14583855867385864], "cos_innovation_negg": [0.2877533733844757, 0.35692280530929565, 0.6591963768005371], "cos_apical_negg": [0.04350179433822632, 0.04822395741939545, -0.14583855867385864], "cos_Ac_negg": [0.28799116611480713, 0.35723939538002014, 0.6615108251571655], "r_norm": [0.32902830839157104, 0.33347609639167786, 0.3232731223106384]}, {"step": 6400, "epoch": 13, "train_loss": 0.010547762736678123, "cos_r_negg": [0.05854140967130661, 0.04904986917972565, 0.008392009884119034], "cos_innovation_negg": [0.31301918625831604, 0.4097648859024048, 0.6412030458450317], "cos_apical_negg": [0.05854140967130661, 0.04904986172914505, 0.008392002433538437], "cos_Ac_negg": [0.31322377920150757, 0.40992504358291626, 0.6410225629806519], "r_norm": [0.32384568452835083, 0.33434537053108215, 0.32019364833831787]}, {"step": 6500, "epoch": 13, "train_loss": 0.02400580793619156, "cos_r_negg": [0.028128569945693016, 0.03145512938499451, -0.14054271578788757], "cos_innovation_negg": [0.29985666275024414, 0.37772876024246216, 0.6384726762771606], "cos_apical_negg": [0.028128571808338165, 0.03145512565970421, -0.14054271578788757], "cos_Ac_negg": [0.2999824285507202, 0.3783789873123169, 0.6384218335151672], "r_norm": [0.30828994512557983, 0.3116949200630188, 0.2992928624153137], "ldr": {"dL_sdil": -0.010763436555862427, "dL_bp": -0.02436085045337677, "ratio": 0.4418333660584685}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9564, "test_loss": 0.20967651138305665}, {"step": 6600, "epoch": 14, "train_loss": 0.0171881802380085, "cos_r_negg": [0.04867755249142647, 0.04371873661875725, -0.034156620502471924], "cos_innovation_negg": [0.3520340025424957, 0.42856040596961975, 0.698298454284668], "cos_apical_negg": [0.04867754876613617, 0.043718740344047546, -0.03415662422776222], "cos_Ac_negg": [0.352585107088089, 0.4290047585964203, 0.6984448432922363], "r_norm": [0.3222231864929199, 0.3260580897331238, 0.3169287145137787]}, {"step": 6700, "epoch": 14, "train_loss": 0.09316784143447876, "cos_r_negg": [0.056634485721588135, 0.05834105238318443, -0.13809379935264587], "cos_innovation_negg": [0.2693135440349579, 0.35252073407173157, 0.7495920658111572], "cos_apical_negg": [0.056634485721588135, 0.05834105610847473, -0.13809381425380707], "cos_Ac_negg": [0.2695942223072052, 0.3542158901691437, 0.7505279779434204], "r_norm": [0.3374485969543457, 0.338696151971817, 0.3270541727542877]}, {"step": 6800, "epoch": 14, "train_loss": 0.11760030686855316, "cos_r_negg": [0.07047758251428604, 0.031747426837682724, -0.15715719759464264], "cos_innovation_negg": [0.28173351287841797, 0.3647039532661438, 0.7331205010414124], "cos_apical_negg": [0.07047758996486664, 0.03174743056297302, -0.15715719759464264], "cos_Ac_negg": [0.2816561460494995, 0.36497336626052856, 0.732842206954956], "r_norm": [0.32171744108200073, 0.3220268487930298, 0.31346920132637024]}, {"step": 6900, "epoch": 14, "train_loss": 0.011903109028935432, "cos_r_negg": [0.05340467020869255, 0.05670728534460068, -0.1993456482887268], "cos_innovation_negg": [0.28802743554115295, 0.36178576946258545, 0.7069607973098755], "cos_apical_negg": [0.05340467020869255, 0.056707289069890976, -0.1993456482887268], "cos_Ac_negg": [0.28918078541755676, 0.3645724952220917, 0.7159875631332397], "r_norm": [0.2909201979637146, 0.29303663969039917, 0.28414618968963623]}, {"step": 7000, "epoch": 14, "train_loss": 0.09724093973636627, "cos_r_negg": [0.04742192476987839, 0.04188825190067291, -0.07511313259601593], "cos_innovation_negg": [0.3258897066116333, 0.3811423182487488, 0.723770260810852], "cos_apical_negg": [0.04742192476987839, 0.04188825190067291, -0.07511313259601593], "cos_Ac_negg": [0.3262384235858917, 0.3813590705394745, 0.7244777083396912], "r_norm": [0.29820430278778076, 0.30305516719818115, 0.2924060523509979], "ldr": {"dL_sdil": -0.01218569278717041, "dL_bp": -0.03188328444957733, "ratio": 0.3821969096829343}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9578, "test_loss": 0.21582944259643555}], "final": {"test_acc": 0.9578, "test_loss": 0.21582944259643555, "wall_s": 24.77858328819275, "cos_r_negg": [0.04536595568060875, 0.04051899164915085, -0.20429718494415283], "cos_innovation_negg": [0.27891454100608826, 0.3399190604686737, 0.8059977889060974], "cos_apical_negg": [0.04536595568060875, 0.04051899164915085, -0.20429719984531403], "cos_Ac_negg": [0.27873682975769043, 0.3405865430831909, 0.8069917559623718]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_matched_s2.json b/results/nuis_ctrl_v1_rho0p2_matched_s2.json new file mode 100644 index 0000000..8a963e7 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_matched_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_matched_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.012568807229399681, 0.03450503945350647, 0.0010404697386547923], "cos_innovation_negg": [0.014475886709988117, 0.03835199773311615, 0.002498562913388014], "cos_apical_negg": [0.012568807229399681, 0.03450503945350647, 0.0010404693894088268], "cos_Ac_negg": [0.01447701919823885, 0.038352131843566895, 0.0024988045915961266], "r_norm": [4.752559661865234, 4.777873992919922, 4.861356735229492], "ldr": {"dL_sdil": -0.03560066223144531, "dL_bp": -0.22435855865478516, "ratio": 0.15867753138057528}}, {"step": 100, "epoch": 0, "train_loss": 0.41686537861824036, "cos_r_negg": [0.20910879969596863, 0.2692984640598297, 0.4817904233932495], "cos_innovation_negg": [0.3817101716995239, 0.44794201850891113, 0.6611666679382324], "cos_apical_negg": [0.20910879969596863, 0.2692984938621521, 0.4817904233932495], "cos_Ac_negg": [0.38167721033096313, 0.44792354106903076, 0.6610803604125977], "r_norm": [1.4123414754867554, 1.4315316677093506, 1.4444916248321533]}, {"step": 200, "epoch": 0, "train_loss": 0.5074231624603271, "cos_r_negg": [0.1471855193376541, 0.1931024193763733, 0.38483262062072754], "cos_innovation_negg": [0.3486558198928833, 0.3702238202095032, 0.7247120141983032], "cos_apical_negg": [0.1471855193376541, 0.1931024193763733, 0.38483262062072754], "cos_Ac_negg": [0.34843605756759644, 0.3701276183128357, 0.7246884107589722], "r_norm": [1.2270182371139526, 1.2435197830200195, 1.2643003463745117]}, {"step": 300, "epoch": 0, "train_loss": 0.37084686756134033, "cos_r_negg": [0.19130156934261322, 0.21340379118919373, 0.2546432912349701], "cos_innovation_negg": [0.42001277208328247, 0.43721991777420044, 0.6830000281333923], "cos_apical_negg": [0.19130158424377441, 0.21340377628803253, 0.2546432912349701], "cos_Ac_negg": [0.41993245482444763, 0.43723857402801514, 0.6830713152885437], "r_norm": [1.0301235914230347, 1.044346570968628, 1.063666582107544]}, {"step": 400, "epoch": 0, "train_loss": 0.26632899045944214, "cos_r_negg": [0.14599348604679108, 0.15959666669368744, 0.22845155000686646], "cos_innovation_negg": [0.36778703331947327, 0.371023565530777, 0.6967618465423584], "cos_apical_negg": [0.14599348604679108, 0.15959666669368744, 0.22845155000686646], "cos_Ac_negg": [0.3677300810813904, 0.3710061311721802, 0.696708083152771], "r_norm": [0.9066268801689148, 0.9191576242446899, 0.9300616979598999]}, {"epoch_end": 0, "step": 469, "test_acc": 0.903, "test_loss": 0.331314372253418}, {"step": 500, "epoch": 1, "train_loss": 0.3867322504520416, "cos_r_negg": [0.15045762062072754, 0.17845138907432556, 0.2959133982658386], "cos_innovation_negg": [0.3879963755607605, 0.4371849298477173, 0.7281054258346558], "cos_apical_negg": [0.15045762062072754, 0.17845138907432556, 0.2959133982658386], "cos_Ac_negg": [0.38791027665138245, 0.4371678829193115, 0.7275727987289429], "r_norm": [0.8599467277526855, 0.8747251033782959, 0.887826681137085], "ldr": {"dL_sdil": -0.028965353965759277, "dL_bp": -0.03613775968551636, "ratio": 0.8015259998911414}}, {"step": 600, "epoch": 1, "train_loss": 0.13958343863487244, "cos_r_negg": [0.1288796365261078, 0.1283964365720749, 0.24638712406158447], "cos_innovation_negg": [0.38277918100357056, 0.40525853633880615, 0.7170636653900146], "cos_apical_negg": [0.12887966632843018, 0.1283964216709137, 0.24638710916042328], "cos_Ac_negg": [0.3828057050704956, 0.40533846616744995, 0.7172442674636841], "r_norm": [0.8258097767829895, 0.8411470055580139, 0.8469644784927368]}, {"step": 700, "epoch": 1, "train_loss": 0.2569888234138489, "cos_r_negg": [0.1412455141544342, 0.17819184064865112, 0.2809038758277893], "cos_innovation_negg": [0.42194926738739014, 0.47824516892433167, 0.811832070350647], "cos_apical_negg": [0.1412455141544342, 0.17819184064865112, 0.2809038758277893], "cos_Ac_negg": [0.4218267798423767, 0.4780724048614502, 0.8120670914649963], "r_norm": [0.872336208820343, 0.8837152719497681, 0.9001341462135315]}, {"step": 800, "epoch": 1, "train_loss": 0.25456687808036804, "cos_r_negg": [0.14802801609039307, 0.14368075132369995, 0.32872581481933594], "cos_innovation_negg": [0.42030400037765503, 0.42957520484924316, 0.794129490852356], "cos_apical_negg": [0.14802801609039307, 0.14368075132369995, 0.32872581481933594], "cos_Ac_negg": [0.4200805425643921, 0.429516077041626, 0.7942253351211548], "r_norm": [0.7989506125450134, 0.8065053820610046, 0.821996808052063]}, {"step": 900, "epoch": 1, "train_loss": 0.34873253107070923, "cos_r_negg": [0.11648806184530258, 0.14995908737182617, 0.5076208710670471], "cos_innovation_negg": [0.4059617221355438, 0.4781762957572937, 0.8521462082862854], "cos_apical_negg": [0.11648805439472198, 0.14995910227298737, 0.5076208710670471], "cos_Ac_negg": [0.40587353706359863, 0.4780960977077484, 0.8521717190742493], "r_norm": [0.851474404335022, 0.8599135279655457, 0.8701728582382202]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9322, "test_loss": 0.26375516510009767}, {"step": 1000, "epoch": 2, "train_loss": 0.1870284080505371, "cos_r_negg": [0.10966353118419647, 0.11476705968379974, 0.35616523027420044], "cos_innovation_negg": [0.3887864947319031, 0.42191082239151, 0.7260174751281738], "cos_apical_negg": [0.10966353118419647, 0.11476706713438034, 0.35616523027420044], "cos_Ac_negg": [0.3884957432746887, 0.4210093021392822, 0.7258301973342896], "r_norm": [0.7246695756912231, 0.7339367866516113, 0.7531286478042603], "ldr": {"dL_sdil": -0.030852317810058594, "dL_bp": -0.049205899238586426, "ratio": 0.6270044504311127}}, {"step": 1100, "epoch": 2, "train_loss": 0.2859516739845276, "cos_r_negg": [0.09784212708473206, 0.15027090907096863, 0.3862214982509613], "cos_innovation_negg": [0.4302035868167877, 0.4694533944129944, 0.8364196419715881], "cos_apical_negg": [0.09784212708473206, 0.15027090907096863, 0.3862214982509613], "cos_Ac_negg": [0.4301931858062744, 0.4694032669067383, 0.8362910747528076], "r_norm": [0.6909522414207458, 0.6969499588012695, 0.7114371061325073]}, {"step": 1200, "epoch": 2, "train_loss": 0.2686888575553894, "cos_r_negg": [0.06694125384092331, 0.10133053362369537, 0.3635830879211426], "cos_innovation_negg": [0.3626210689544678, 0.42960411310195923, 0.7312387228012085], "cos_apical_negg": [0.06694125384092331, 0.10133052617311478, 0.3635830581188202], "cos_Ac_negg": [0.3625844120979309, 0.4294888377189636, 0.7304561138153076], "r_norm": [0.5011560320854187, 0.5067845582962036, 0.5153244733810425]}, {"step": 1300, "epoch": 2, "train_loss": 0.233536034822464, "cos_r_negg": [0.13068357110023499, 0.13832031190395355, 0.44007012248039246], "cos_innovation_negg": [0.4261730909347534, 0.4654122591018677, 0.8375815153121948], "cos_apical_negg": [0.13068357110023499, 0.13832029700279236, 0.44007015228271484], "cos_Ac_negg": [0.426139771938324, 0.4653242230415344, 0.8375306725502014], "r_norm": [0.7261972427368164, 0.7375327348709106, 0.7505894899368286]}, {"step": 1400, "epoch": 2, "train_loss": 0.3284955620765686, "cos_r_negg": [0.12551483511924744, 0.15524140000343323, 0.4092218279838562], "cos_innovation_negg": [0.3842164874076843, 0.4378356337547302, 0.7856218814849854], "cos_apical_negg": [0.12551483511924744, 0.15524140000343323, 0.4092218279838562], "cos_Ac_negg": [0.3841329514980316, 0.4376981854438782, 0.7854340672492981], "r_norm": [0.7306188941001892, 0.7402686476707458, 0.7559319734573364]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9389, "test_loss": 0.23942671127319337}, {"step": 1500, "epoch": 3, "train_loss": 0.19460006058216095, "cos_r_negg": [0.11979489773511887, 0.1436571478843689, 0.3284034729003906], "cos_innovation_negg": [0.3952661156654358, 0.46787145733833313, 0.8080404996871948], "cos_apical_negg": [0.11979489773511887, 0.1436571478843689, 0.3284034729003906], "cos_Ac_negg": [0.3951954245567322, 0.46767285466194153, 0.8072835206985474], "r_norm": [0.618583083152771, 0.6279668211936951, 0.6388745307922363], "ldr": {"dL_sdil": -0.015642642974853516, "dL_bp": -0.02876603603363037, "ratio": 0.5437886178187952}}, {"step": 1600, "epoch": 3, "train_loss": 0.3088424801826477, "cos_r_negg": [0.10899022221565247, 0.1483244001865387, 0.4355407953262329], "cos_innovation_negg": [0.39527010917663574, 0.49079176783561707, 0.8139268159866333], "cos_apical_negg": [0.10899022221565247, 0.1483244001865387, 0.4355407953262329], "cos_Ac_negg": [0.39524132013320923, 0.4907287657260895, 0.8137434720993042], "r_norm": [0.6653609275817871, 0.6724896430969238, 0.6789262294769287]}, {"step": 1700, "epoch": 3, "train_loss": 0.10894760489463806, "cos_r_negg": [0.10433030873537064, 0.13236722350120544, 0.4496546983718872], "cos_innovation_negg": [0.41957804560661316, 0.5137567520141602, 0.7548569440841675], "cos_apical_negg": [0.10433030873537064, 0.13236720860004425, 0.4496547281742096], "cos_Ac_negg": [0.4192151427268982, 0.5135338306427002, 0.7546485066413879], "r_norm": [0.5623418688774109, 0.5709551572799683, 0.579412579536438]}, {"step": 1800, "epoch": 3, "train_loss": 0.04571342468261719, "cos_r_negg": [0.09522800147533417, 0.1473408043384552, 0.3995060920715332], "cos_innovation_negg": [0.3593822419643402, 0.45815226435661316, 0.7739691734313965], "cos_apical_negg": [0.09522800147533417, 0.147340789437294, 0.3995060920715332], "cos_Ac_negg": [0.3595648407936096, 0.4577977657318115, 0.77255779504776], "r_norm": [0.6105034947395325, 0.6216210126876831, 0.6287780404090881]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9451, "test_loss": 0.2317189651489258}, {"step": 1900, "epoch": 4, "train_loss": 0.1580403596162796, "cos_r_negg": [0.08415976166725159, 0.09746617823839188, 0.20728908479213715], "cos_innovation_negg": [0.38979971408843994, 0.48892247676849365, 0.7774503827095032], "cos_apical_negg": [0.08415976166725159, 0.09746617078781128, 0.20728906989097595], "cos_Ac_negg": [0.389920711517334, 0.4887709617614746, 0.7771754264831543], "r_norm": [0.46975407004356384, 0.47606760263442993, 0.4823441803455353]}, {"step": 2000, "epoch": 4, "train_loss": 0.19678255915641785, "cos_r_negg": [0.08281712979078293, 0.12148278951644897, 0.29431283473968506], "cos_innovation_negg": [0.3804301619529724, 0.4397244453430176, 0.7883623242378235], "cos_apical_negg": [0.08281712234020233, 0.12148278951644897, 0.29431283473968506], "cos_Ac_negg": [0.3800572156906128, 0.4395955502986908, 0.788510799407959], "r_norm": [0.5446688532829285, 0.5521743297576904, 0.5632345676422119], "ldr": {"dL_sdil": -0.013858526945114136, "dL_bp": -0.02352052927017212, "ratio": 0.5892098254221267}}, {"step": 2100, "epoch": 4, "train_loss": 0.13737311959266663, "cos_r_negg": [0.09908808022737503, 0.13024359941482544, 0.39038828015327454], "cos_innovation_negg": [0.3986241817474365, 0.5013235211372375, 0.8444226980209351], "cos_apical_negg": [0.09908808022737503, 0.13024359941482544, 0.39038828015327454], "cos_Ac_negg": [0.39859622716903687, 0.5009471774101257, 0.8442931175231934], "r_norm": [0.5882489681243896, 0.5957415699958801, 0.6045143604278564]}, {"step": 2200, "epoch": 4, "train_loss": 0.19626152515411377, "cos_r_negg": [0.07650864124298096, 0.10710731893777847, 0.39824801683425903], "cos_innovation_negg": [0.34140971302986145, 0.4399786591529846, 0.7429449558258057], "cos_apical_negg": [0.07650864124298096, 0.10710731148719788, 0.39824801683425903], "cos_Ac_negg": [0.3411194980144501, 0.43962615728378296, 0.7430374622344971], "r_norm": [0.6595460772514343, 0.6734328866004944, 0.6698944568634033]}, {"step": 2300, "epoch": 4, "train_loss": 0.3648568093776703, "cos_r_negg": [0.09674303978681564, 0.059799689799547195, 0.42855992913246155], "cos_innovation_negg": [0.3925301432609558, 0.4957837760448456, 0.745842695236206], "cos_apical_negg": [0.09674303978681564, 0.0597996860742569, 0.42855992913246155], "cos_Ac_negg": [0.3922707140445709, 0.4958387613296509, 0.7451333403587341], "r_norm": [0.4760410189628601, 0.48340457677841187, 0.4886839985847473]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9467, "test_loss": 0.21527346267700195}, {"step": 2400, "epoch": 5, "train_loss": 0.23511075973510742, "cos_r_negg": [0.11027809977531433, 0.09582982957363129, 0.25784823298454285], "cos_innovation_negg": [0.3743487596511841, 0.4688841998577118, 0.8296862840652466], "cos_apical_negg": [0.11027809977531433, 0.09582984447479248, 0.25784820318222046], "cos_Ac_negg": [0.37426698207855225, 0.46891024708747864, 0.8296178579330444], "r_norm": [0.5222523808479309, 0.531295895576477, 0.5303417444229126]}, {"step": 2500, "epoch": 5, "train_loss": 0.1752617359161377, "cos_r_negg": [0.09847832471132278, 0.06718166917562485, 0.17765995860099792], "cos_innovation_negg": [0.3913056552410126, 0.4618316888809204, 0.8533058166503906], "cos_apical_negg": [0.09847832471132278, 0.06718166172504425, 0.17765995860099792], "cos_Ac_negg": [0.39109960198402405, 0.46172210574150085, 0.8523133397102356], "r_norm": [0.43065425753593445, 0.43950438499450684, 0.4415263235569], "ldr": {"dL_sdil": -0.005804121494293213, "dL_bp": -0.017602145671844482, "ratio": 0.3297394307773056}}, {"step": 2600, "epoch": 5, "train_loss": 0.13911238312721252, "cos_r_negg": [0.07710015773773193, 0.08898136019706726, 0.22105585038661957], "cos_innovation_negg": [0.41214719414711, 0.5028194189071655, 0.8794364929199219], "cos_apical_negg": [0.07710016518831253, 0.08898136764764786, 0.22105585038661957], "cos_Ac_negg": [0.4114411473274231, 0.5025098323822021, 0.8802287578582764], "r_norm": [0.43281257152557373, 0.439570814371109, 0.4430481791496277]}, {"step": 2700, "epoch": 5, "train_loss": 0.17145106196403503, "cos_r_negg": [0.09005577117204666, 0.09019629657268524, 0.22444583475589752], "cos_innovation_negg": [0.40291374921798706, 0.5120317339897156, 0.8890208005905151], "cos_apical_negg": [0.09005577117204666, 0.09019629657268524, 0.22444583475589752], "cos_Ac_negg": [0.40267127752304077, 0.5120372176170349, 0.889025092124939], "r_norm": [0.4981767535209656, 0.5076959133148193, 0.5086133480072021]}, {"step": 2800, "epoch": 5, "train_loss": 0.31462931632995605, "cos_r_negg": [0.10539068281650543, 0.11976125836372375, 0.22073815762996674], "cos_innovation_negg": [0.3493485450744629, 0.48015284538269043, 0.8482993841171265], "cos_apical_negg": [0.10539068281650543, 0.11976125091314316, 0.22073815762996674], "cos_Ac_negg": [0.34916335344314575, 0.4801478385925293, 0.8480631113052368], "r_norm": [0.5429097414016724, 0.5564067363739014, 0.5528233051300049]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9439, "test_loss": 0.24497178802490235}, {"step": 2900, "epoch": 6, "train_loss": 0.03512556105852127, "cos_r_negg": [0.06368514895439148, 0.06456251442432404, 0.1814042180776596], "cos_innovation_negg": [0.37936916947364807, 0.5022355914115906, 0.8352999091148376], "cos_apical_negg": [0.06368514895439148, 0.06456251442432404, 0.1814042329788208], "cos_Ac_negg": [0.3791946768760681, 0.5023296475410461, 0.8349480628967285], "r_norm": [0.3785375952720642, 0.3863418996334076, 0.388815313577652]}, {"step": 3000, "epoch": 6, "train_loss": 0.37099123001098633, "cos_r_negg": [0.07447345554828644, 0.07731825858354568, 0.2265074998140335], "cos_innovation_negg": [0.34065717458724976, 0.4763009548187256, 0.9335079789161682], "cos_apical_negg": [0.07447345554828644, 0.07731825858354568, 0.2265075147151947], "cos_Ac_negg": [0.3405773341655731, 0.4761340618133545, 0.9335340261459351], "r_norm": [0.4048287868499756, 0.4124051332473755, 0.4092932343482971], "ldr": {"dL_sdil": -0.010610625147819519, "dL_bp": -0.023834601044654846, "ratio": 0.44517737586377853}}, {"step": 3100, "epoch": 6, "train_loss": 0.16707779467105865, "cos_r_negg": [0.08212460577487946, 0.098589226603508, 0.19514133036136627], "cos_innovation_negg": [0.34616410732269287, 0.4842938780784607, 0.8460375070571899], "cos_apical_negg": [0.08212461322546005, 0.0985892191529274, 0.19514131546020508], "cos_Ac_negg": [0.3459634780883789, 0.4841005802154541, 0.8459081649780273], "r_norm": [0.44524234533309937, 0.4536617696285248, 0.45436015725135803]}, {"step": 3200, "epoch": 6, "train_loss": 0.14609743654727936, "cos_r_negg": [0.05026909336447716, 0.08298943191766739, 0.07404734939336777], "cos_innovation_negg": [0.30734139680862427, 0.3544612228870392, 0.8999783992767334], "cos_apical_negg": [0.05026909336447716, 0.08298943191766739, 0.07404734939336777], "cos_Ac_negg": [0.3073403239250183, 0.35445746779441833, 0.8998667001724243], "r_norm": [0.38567402958869934, 0.3945664167404175, 0.39722272753715515]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9462, "test_loss": 0.2586587181091309}, {"step": 3300, "epoch": 7, "train_loss": 0.10547500103712082, "cos_r_negg": [0.056288231164216995, 0.04224563390016556, 0.1539236605167389], "cos_innovation_negg": [0.3357273042201996, 0.3908730447292328, 0.8601452112197876], "cos_apical_negg": [0.0562882274389267, 0.042245637625455856, 0.1539236605167389], "cos_Ac_negg": [0.3356916904449463, 0.3908854126930237, 0.8599631190299988], "r_norm": [0.42344582080841064, 0.4351963400840759, 0.4372221827507019]}, {"step": 3400, "epoch": 7, "train_loss": 0.17714954912662506, "cos_r_negg": [0.09304238855838776, 0.06626348197460175, 0.2179740071296692], "cos_innovation_negg": [0.3474711775779724, 0.4196842908859253, 0.7574760317802429], "cos_apical_negg": [0.09304238855838776, 0.06626348197460175, 0.2179740071296692], "cos_Ac_negg": [0.3475801348686218, 0.41989079117774963, 0.7574885487556458], "r_norm": [0.5394315123558044, 0.5502663254737854, 0.5532580614089966]}, {"step": 3500, "epoch": 7, "train_loss": 0.17387335002422333, "cos_r_negg": [0.07497861236333847, 0.1394818127155304, 0.20803505182266235], "cos_innovation_negg": [0.3240651488304138, 0.4542658030986786, 0.8671237230300903], "cos_apical_negg": [0.07497861236333847, 0.1394818127155304, 0.20803505182266235], "cos_Ac_negg": [0.324073851108551, 0.4542570114135742, 0.8668864369392395], "r_norm": [0.4373140335083008, 0.4463554620742798, 0.4412674307823181], "ldr": {"dL_sdil": -0.015811651945114136, "dL_bp": -0.03220531344413757, "ratio": 0.49096407561878197}}, {"step": 3600, "epoch": 7, "train_loss": 0.06002667546272278, "cos_r_negg": [0.052177365869283676, 0.06647840142250061, 0.0639357715845108], "cos_innovation_negg": [0.33399060368537903, 0.42625439167022705, 0.8619518876075745], "cos_apical_negg": [0.052177365869283676, 0.06647840142250061, 0.0639357715845108], "cos_Ac_negg": [0.3339133560657501, 0.42644232511520386, 0.8618908524513245], "r_norm": [0.5129511952400208, 0.5256119966506958, 0.524181604385376]}, {"step": 3700, "epoch": 7, "train_loss": 0.09367530047893524, "cos_r_negg": [0.05952652543783188, 0.06526409089565277, 0.1319093108177185], "cos_innovation_negg": [0.3227085471153259, 0.41077321767807007, 0.8404790163040161], "cos_apical_negg": [0.05952652543783188, 0.06526409834623337, 0.1319093108177185], "cos_Ac_negg": [0.322771281003952, 0.41081634163856506, 0.8405077457427979], "r_norm": [0.4133445620536804, 0.4207608103752136, 0.4252481460571289]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9453, "test_loss": 0.2556048843383789}, {"step": 3800, "epoch": 8, "train_loss": 0.10329148173332214, "cos_r_negg": [0.05341097339987755, 0.07640786468982697, 0.11473394930362701], "cos_innovation_negg": [0.2759247422218323, 0.3585544526576996, 0.7997012734413147], "cos_apical_negg": [0.05341096967458725, 0.07640786468982697, 0.11473394930362701], "cos_Ac_negg": [0.275887131690979, 0.3585386872291565, 0.799127459526062], "r_norm": [0.44362640380859375, 0.45211684703826904, 0.45843958854675293]}, {"step": 3900, "epoch": 8, "train_loss": 0.35718321800231934, "cos_r_negg": [0.049845509231090546, 0.10246105492115021, 0.1079498678445816], "cos_innovation_negg": [0.3130926489830017, 0.39905768632888794, 0.8223754167556763], "cos_apical_negg": [0.049845509231090546, 0.1024610698223114, 0.1079498827457428], "cos_Ac_negg": [0.31375807523727417, 0.4004639685153961, 0.8252338171005249], "r_norm": [0.4643113613128662, 0.47499728202819824, 0.481009840965271]}, {"step": 4000, "epoch": 8, "train_loss": 0.2836810350418091, "cos_r_negg": [0.05736468359827995, 0.04499705880880356, -0.0678829625248909], "cos_innovation_negg": [0.27919501066207886, 0.35739651322364807, 0.8778584599494934], "cos_apical_negg": [0.05736468359827995, 0.04499705880880356, -0.0678829699754715], "cos_Ac_negg": [0.2792021632194519, 0.3576590418815613, 0.8780800104141235], "r_norm": [0.3771244287490845, 0.38823267817497253, 0.3904397487640381], "ldr": {"dL_sdil": -0.009959444403648376, "dL_bp": -0.02218519151210785, "ratio": 0.4489230754763998}}, {"step": 4100, "epoch": 8, "train_loss": 0.042821288108825684, "cos_r_negg": [0.058858368545770645, 0.06586670875549316, 0.22785906493663788], "cos_innovation_negg": [0.29042911529541016, 0.39218538999557495, 0.828285813331604], "cos_apical_negg": [0.05885836482048035, 0.06586670875549316, 0.22785906493663788], "cos_Ac_negg": [0.29045259952545166, 0.39242100715637207, 0.8278027176856995], "r_norm": [0.4354664087295532, 0.4457669258117676, 0.4443976581096649]}, {"step": 4200, "epoch": 8, "train_loss": 0.5092855095863342, "cos_r_negg": [0.056518882513046265, 0.11766859143972397, 0.27322641015052795], "cos_innovation_negg": [0.3196059465408325, 0.4284757971763611, 0.8559291958808899], "cos_apical_negg": [0.056518882513046265, 0.11766858398914337, 0.27322641015052795], "cos_Ac_negg": [0.319627046585083, 0.428483784198761, 0.8559736609458923], "r_norm": [0.5603938698768616, 0.5704473257064819, 0.5819420218467712]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9522, "test_loss": 0.22193515548706055}, {"step": 4300, "epoch": 9, "train_loss": 0.18095043301582336, "cos_r_negg": [0.04524961858987808, 0.06241091713309288, 0.011419588699936867], "cos_innovation_negg": [0.2987020015716553, 0.4112701416015625, 0.8658174276351929], "cos_apical_negg": [0.04524961858987808, 0.06241092458367348, 0.011419599875807762], "cos_Ac_negg": [0.29874253273010254, 0.4113137125968933, 0.8660538196563721], "r_norm": [0.36676833033561707, 0.37489771842956543, 0.3765292167663574]}, {"step": 4400, "epoch": 9, "train_loss": 0.05341877415776253, "cos_r_negg": [0.04234806075692177, 0.028406698256731033, 0.06240399181842804], "cos_innovation_negg": [0.26289868354797363, 0.32441675662994385, 0.8132338523864746], "cos_apical_negg": [0.04234806075692177, 0.028406694531440735, 0.06240399181842804], "cos_Ac_negg": [0.26288479566574097, 0.324606716632843, 0.8131864070892334], "r_norm": [0.34485629200935364, 0.352750688791275, 0.354925274848938]}, {"step": 4500, "epoch": 9, "train_loss": 0.28681567311286926, "cos_r_negg": [0.05675864964723587, 0.07759023457765579, 0.24251818656921387], "cos_innovation_negg": [0.3238711953163147, 0.3914225697517395, 0.7244126796722412], "cos_apical_negg": [0.05675865337252617, 0.07759023457765579, 0.24251818656921387], "cos_Ac_negg": [0.3239164352416992, 0.3916572630405426, 0.7245314121246338], "r_norm": [0.42983636260032654, 0.43696096539497375, 0.44225671887397766], "ldr": {"dL_sdil": -0.023283734917640686, "dL_bp": -0.058171361684799194, "ratio": 0.40026112924437485}}, {"step": 4600, "epoch": 9, "train_loss": 0.16406282782554626, "cos_r_negg": [0.06896272301673889, 0.08038609474897385, 0.19771063327789307], "cos_innovation_negg": [0.30861198902130127, 0.4007579982280731, 0.8681092262268066], "cos_apical_negg": [0.06896272301673889, 0.08038608729839325, 0.19771063327789307], "cos_Ac_negg": [0.30873942375183105, 0.4010622799396515, 0.8685113191604614], "r_norm": [0.5101726651191711, 0.5222765207290649, 0.5313335657119751]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9517, "test_loss": 0.22202390060424804}, {"step": 4700, "epoch": 10, "train_loss": 0.23987311124801636, "cos_r_negg": [0.05266813933849335, 0.10070398449897766, 0.10234659165143967], "cos_innovation_negg": [0.31556618213653564, 0.41748785972595215, 0.8992655277252197], "cos_apical_negg": [0.05266813933849335, 0.10070399194955826, 0.10234659910202026], "cos_Ac_negg": [0.31568941473960876, 0.4175994098186493, 0.8997846841812134], "r_norm": [0.3324055075645447, 0.34350085258483887, 0.34051284193992615]}, {"step": 4800, "epoch": 10, "train_loss": 0.12037467211484909, "cos_r_negg": [0.043553147464990616, 0.04972739517688751, 0.022600214928388596], "cos_innovation_negg": [0.2953176200389862, 0.39072006940841675, 0.7780461311340332], "cos_apical_negg": [0.04355314373970032, 0.04972739517688751, 0.022600218653678894], "cos_Ac_negg": [0.2957000732421875, 0.3912743330001831, 0.7785463929176331], "r_norm": [0.2830619215965271, 0.29055994749069214, 0.29010844230651855]}, {"step": 4900, "epoch": 10, "train_loss": 0.11967198550701141, "cos_r_negg": [0.033858709037303925, 0.06222856044769287, 0.060803718864917755], "cos_innovation_negg": [0.26039254665374756, 0.3370136618614197, 0.8748444318771362], "cos_apical_negg": [0.033858705312013626, 0.06222856044769287, 0.06080370396375656], "cos_Ac_negg": [0.26048630475997925, 0.3371487259864807, 0.8747726082801819], "r_norm": [0.40528276562690735, 0.41331154108047485, 0.41936299204826355]}, {"step": 5000, "epoch": 10, "train_loss": 0.06904764473438263, "cos_r_negg": [0.03591812402009964, 0.045499902218580246, 0.10747598111629486], "cos_innovation_negg": [0.29342421889305115, 0.41245999932289124, 0.8740525245666504], "cos_apical_negg": [0.03591813147068024, 0.045499905943870544, 0.10747598111629486], "cos_Ac_negg": [0.29350489377975464, 0.41271156072616577, 0.8742175102233887], "r_norm": [0.32368016242980957, 0.3318740129470825, 0.33392614126205444], "ldr": {"dL_sdil": -0.009457185864448547, "dL_bp": -0.03640947490930557, "ratio": 0.2597451868780307}}, {"step": 5100, "epoch": 10, "train_loss": 0.2273128628730774, "cos_r_negg": [0.05880279839038849, 0.03401549160480499, 0.20626923441886902], "cos_innovation_negg": [0.28785860538482666, 0.3812047839164734, 0.7261700630187988], "cos_apical_negg": [0.05880279839038849, 0.03401549160480499, 0.20626923441886902], "cos_Ac_negg": [0.28785449266433716, 0.3814231753349304, 0.7272638082504272], "r_norm": [0.3790310025215149, 0.3893599510192871, 0.3848729133605957]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9504, "test_loss": 0.2372825439453125}, {"step": 5200, "epoch": 11, "train_loss": 0.0498318113386631, "cos_r_negg": [0.007242933381348848, 0.07718637585639954, 0.09968007355928421], "cos_innovation_negg": [0.2896762490272522, 0.3871708810329437, 0.827749490737915], "cos_apical_negg": [0.007242933847010136, 0.07718637585639954, 0.09968006610870361], "cos_Ac_negg": [0.29003095626831055, 0.3872098922729492, 0.8282887935638428], "r_norm": [0.27251529693603516, 0.2822016775608063, 0.2801424264907837]}, {"step": 5300, "epoch": 11, "train_loss": 0.05062280595302582, "cos_r_negg": [0.03909166157245636, 0.03862752392888069, 0.0009403862059116364], "cos_innovation_negg": [0.2808224856853485, 0.359735369682312, 0.8595982789993286], "cos_apical_negg": [0.03909166157245636, 0.03862752020359039, 0.0009403880685567856], "cos_Ac_negg": [0.281141996383667, 0.36012542247772217, 0.861784815788269], "r_norm": [0.32579702138900757, 0.33651769161224365, 0.33541446924209595]}, {"step": 5400, "epoch": 11, "train_loss": 0.23429664969444275, "cos_r_negg": [0.027438657358288765, 0.015929793938994408, -0.021444575861096382], "cos_innovation_negg": [0.21383386850357056, 0.3027658462524414, 0.82911616563797], "cos_apical_negg": [0.027438659220933914, 0.01592979207634926, -0.02144458144903183], "cos_Ac_negg": [0.21365989744663239, 0.3034135103225708, 0.8291270732879639], "r_norm": [0.3103151321411133, 0.31865739822387695, 0.3214034140110016]}, {"step": 5500, "epoch": 11, "train_loss": 0.1625845730304718, "cos_r_negg": [0.045043911784887314, 0.049678415060043335, 0.053376663476228714], "cos_innovation_negg": [0.27249467372894287, 0.3523178696632385, 0.7552767992019653], "cos_apical_negg": [0.045043908059597015, 0.049678415060043335, 0.05337667837738991], "cos_Ac_negg": [0.2724238932132721, 0.35254430770874023, 0.7553820610046387], "r_norm": [0.34477922320365906, 0.35606497526168823, 0.35589268803596497], "ldr": {"dL_sdil": -0.018144384026527405, "dL_bp": -0.05131293833255768, "ratio": 0.35360251461208814}}, {"step": 5600, "epoch": 11, "train_loss": 0.12291401624679565, "cos_r_negg": [0.05214955657720566, 0.04136645793914795, 0.1848919689655304], "cos_innovation_negg": [0.24350149929523468, 0.4012623131275177, 0.7893986105918884], "cos_apical_negg": [0.05214955285191536, 0.04136645793914795, 0.1848919689655304], "cos_Ac_negg": [0.24356074631214142, 0.40161749720573425, 0.7904373407363892], "r_norm": [0.38925984501838684, 0.4000493884086609, 0.39850032329559326]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9541, "test_loss": 0.23703841400146483}, {"step": 5700, "epoch": 12, "train_loss": 0.042293816804885864, "cos_r_negg": [0.04400692135095596, 0.07849792391061783, 0.09001844376325607], "cos_innovation_negg": [0.33002060651779175, 0.41843074560165405, 0.742067813873291], "cos_apical_negg": [0.04400692135095596, 0.07849793136119843, 0.09001845121383667], "cos_Ac_negg": [0.3301641643047333, 0.4186265468597412, 0.7426013350486755], "r_norm": [0.3201180100440979, 0.32971805334091187, 0.3304041028022766]}, {"step": 5800, "epoch": 12, "train_loss": 0.012315645813941956, "cos_r_negg": [0.028804663568735123, 0.07033725827932358, 0.05612102895975113], "cos_innovation_negg": [0.3163420259952545, 0.39414548873901367, 0.7633299827575684], "cos_apical_negg": [0.02880466729402542, 0.07033725827932358, 0.05612102895975113], "cos_Ac_negg": [0.3164098560810089, 0.3943769037723541, 0.7628835439682007], "r_norm": [0.3078469932079315, 0.3178245425224304, 0.3166249990463257]}, {"step": 5900, "epoch": 12, "train_loss": 0.16876712441444397, "cos_r_negg": [0.008157514035701752, 0.06027689576148987, 0.18386754393577576], "cos_innovation_negg": [0.27397263050079346, 0.37220025062561035, 0.8863725066184998], "cos_apical_negg": [0.008157514967024326, 0.06027689576148987, 0.18386754393577576], "cos_Ac_negg": [0.2744218111038208, 0.3725711703300476, 0.8862937092781067], "r_norm": [0.28205597400665283, 0.2917092442512512, 0.29058098793029785]}, {"step": 6000, "epoch": 12, "train_loss": 0.07225620001554489, "cos_r_negg": [0.024747641757130623, 0.04471999406814575, -0.0409160852432251], "cos_innovation_negg": [0.2468918412923813, 0.3329704999923706, 0.7443671226501465], "cos_apical_negg": [0.024747643619775772, 0.044719986617565155, -0.04091609641909599], "cos_Ac_negg": [0.2471471130847931, 0.3332672715187073, 0.7443922758102417], "r_norm": [0.2492891401052475, 0.25843819975852966, 0.2571605443954468], "ldr": {"dL_sdil": -0.009040534496307373, "dL_bp": -0.03329023718833923, "ratio": 0.2715671397941873}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9524, "test_loss": 0.2407943161010742}, {"step": 6100, "epoch": 13, "train_loss": 0.14530141651630402, "cos_r_negg": [0.04541394114494324, 0.057854704558849335, 0.16725577414035797], "cos_innovation_negg": [0.3677285313606262, 0.47214144468307495, 0.7238172888755798], "cos_apical_negg": [0.045413944870233536, 0.057854700833559036, 0.16725577414035797], "cos_Ac_negg": [0.3687947392463684, 0.4732314944267273, 0.7256034016609192], "r_norm": [0.3035060167312622, 0.3135230839252472, 0.31212127208709717]}, {"step": 6200, "epoch": 13, "train_loss": 0.06230597198009491, "cos_r_negg": [0.029485434293746948, 0.022142738103866577, 0.13790401816368103], "cos_innovation_negg": [0.31515902280807495, 0.41734254360198975, 0.8308672904968262], "cos_apical_negg": [0.029485434293746948, 0.022142736241221428, 0.13790401816368103], "cos_Ac_negg": [0.31533265113830566, 0.4177112877368927, 0.8314176797866821], "r_norm": [0.26477083563804626, 0.27269890904426575, 0.2718859314918518]}, {"step": 6300, "epoch": 13, "train_loss": 0.05405279994010925, "cos_r_negg": [0.03785037249326706, 0.059913620352745056, 0.09468258917331696], "cos_innovation_negg": [0.34503185749053955, 0.4215003550052643, 0.758604884147644], "cos_apical_negg": [0.03785036876797676, 0.05991361662745476, 0.09468260407447815], "cos_Ac_negg": [0.34533849358558655, 0.42184269428253174, 0.758166491985321], "r_norm": [0.3182007372379303, 0.3287730813026428, 0.3284202516078949]}, {"step": 6400, "epoch": 13, "train_loss": 0.028042662888765335, "cos_r_negg": [0.06076284870505333, 0.030982065945863724, 0.0843062698841095], "cos_innovation_negg": [0.31339356303215027, 0.42726075649261475, 0.8195279240608215], "cos_apical_negg": [0.06076285243034363, 0.030982069671154022, 0.0843062698841095], "cos_Ac_negg": [0.3135431408882141, 0.4280187487602234, 0.8196619749069214], "r_norm": [0.2934676706790924, 0.3019433915615082, 0.3032127320766449]}, {"step": 6500, "epoch": 13, "train_loss": 0.026540206745266914, "cos_r_negg": [0.03082185797393322, 0.050485093146562576, 0.19748613238334656], "cos_innovation_negg": [0.32724714279174805, 0.42824146151542664, 0.7925037741661072], "cos_apical_negg": [0.030821863561868668, 0.050485093146562576, 0.19748613238334656], "cos_Ac_negg": [0.32921838760375977, 0.4296613037586212, 0.7946335077285767], "r_norm": [0.28750476241111755, 0.2969520688056946, 0.2947912812232971], "ldr": {"dL_sdil": -0.011314094066619873, "dL_bp": -0.03042103350162506, "ratio": 0.37191682084093186}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9513, "test_loss": 0.23059152603149413}, {"step": 6600, "epoch": 14, "train_loss": 0.07924970984458923, "cos_r_negg": [0.047696471214294434, 0.055230170488357544, 0.20281769335269928], "cos_innovation_negg": [0.3046934902667999, 0.3822438418865204, 0.600570797920227], "cos_apical_negg": [0.047696471214294434, 0.05523017421364784, 0.20281767845153809], "cos_Ac_negg": [0.306087851524353, 0.3829191327095032, 0.6029512882232666], "r_norm": [0.4235853850841522, 0.43880459666252136, 0.4361393451690674]}, {"step": 6700, "epoch": 14, "train_loss": 0.11516257375478745, "cos_r_negg": [0.044589728116989136, 0.08143305033445358, 0.17623205482959747], "cos_innovation_negg": [0.2617274820804596, 0.33344557881355286, 0.6897778511047363], "cos_apical_negg": [0.044589728116989136, 0.08143304288387299, 0.17623203992843628], "cos_Ac_negg": [0.2618792653083801, 0.3339275121688843, 0.6897225975990295], "r_norm": [0.37319427728652954, 0.38359948992729187, 0.3834499716758728]}, {"step": 6800, "epoch": 14, "train_loss": 0.1783713400363922, "cos_r_negg": [0.03542661666870117, 0.07836959511041641, -0.02201629802584648], "cos_innovation_negg": [0.2853891849517822, 0.389090359210968, 0.8815319538116455], "cos_apical_negg": [0.03542662039399147, 0.07836959511041641, -0.02201629802584648], "cos_Ac_negg": [0.28554609417915344, 0.3893526792526245, 0.8816879987716675], "r_norm": [0.3684077262878418, 0.38319459557533264, 0.373880535364151]}, {"step": 6900, "epoch": 14, "train_loss": 0.12244295328855515, "cos_r_negg": [0.0395997054874897, 0.02758469618856907, -0.02063397690653801], "cos_innovation_negg": [0.2926270663738251, 0.4145168960094452, 0.940355122089386], "cos_apical_negg": [0.0395997017621994, 0.027584698051214218, -0.020633984357118607], "cos_Ac_negg": [0.2927078604698181, 0.41444075107574463, 0.941655158996582], "r_norm": [0.32405373454093933, 0.3354256749153137, 0.3306483030319214]}, {"step": 7000, "epoch": 14, "train_loss": 0.07864569127559662, "cos_r_negg": [0.029273169115185738, 0.038981493562459946, 0.19176092743873596], "cos_innovation_negg": [0.3026566207408905, 0.4366740882396698, 0.8388965725898743], "cos_apical_negg": [0.02927316725254059, 0.03898148983716965, 0.19176094233989716], "cos_Ac_negg": [0.30447787046432495, 0.43892902135849, 0.8439494371414185], "r_norm": [0.31825825572013855, 0.3290184438228607, 0.32477784156799316], "ldr": {"dL_sdil": -0.013482391834259033, "dL_bp": -0.038292258977890015, "ratio": 0.3520918377274054}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.96, "test_loss": 0.20232843093872072}], "final": {"test_acc": 0.96, "test_loss": 0.20232843093872072, "wall_s": 24.8452787399292, "cos_r_negg": [0.011547637172043324, 0.014467829838395119, 0.06745955348014832], "cos_innovation_negg": [0.2807725667953491, 0.3743821382522583, 0.8533952236175537], "cos_apical_negg": [0.011547635309398174, 0.01446782797574997, 0.06745956093072891], "cos_Ac_negg": [0.2816218137741089, 0.3756491541862488, 0.8537510633468628]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_matched_s3.json b/results/nuis_ctrl_v1_rho0p2_matched_s3.json new file mode 100644 index 0000000..6405177 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_matched_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_matched_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.02036810666322708, 0.011795876547694206, 0.0021237998735159636], "cos_innovation_negg": [-0.018753841519355774, 0.014596175402402878, 0.005968508310616016], "cos_apical_negg": [-0.02036810666322708, 0.011795875616371632, 0.0021237991750240326], "cos_Ac_negg": [-0.01875452697277069, 0.014595884829759598, 0.0059679653495550156], "r_norm": [4.867480278015137, 4.757315635681152, 4.860472202301025], "ldr": {"dL_sdil": 0.02511310577392578, "dL_bp": -0.2193007469177246, "ratio": -0.11451445618353276}}, {"step": 100, "epoch": 0, "train_loss": 0.4400538206100464, "cos_r_negg": [0.1756768822669983, 0.20653212070465088, 0.3905373811721802], "cos_innovation_negg": [0.31465643644332886, 0.3354240655899048, 0.7031124234199524], "cos_apical_negg": [0.1756768822669983, 0.20653212070465088, 0.3905373811721802], "cos_Ac_negg": [0.3145856261253357, 0.33537915349006653, 0.7031100392341614], "r_norm": [1.5759222507476807, 1.5576527118682861, 1.563110589981079]}, {"step": 200, "epoch": 0, "train_loss": 0.5143619775772095, "cos_r_negg": [0.15921232104301453, 0.17813655734062195, 0.24890244007110596], "cos_innovation_negg": [0.3653538227081299, 0.3882541060447693, 0.5765550136566162], "cos_apical_negg": [0.15921230614185333, 0.17813655734062195, 0.24890244007110596], "cos_Ac_negg": [0.36479735374450684, 0.38796794414520264, 0.5766723155975342], "r_norm": [1.2929868698120117, 1.2643277645111084, 1.292585849761963]}, {"step": 300, "epoch": 0, "train_loss": 0.47719043493270874, "cos_r_negg": [0.1595289707183838, 0.19067716598510742, 0.4023829400539398], "cos_innovation_negg": [0.32857978343963623, 0.40665096044540405, 0.7169294357299805], "cos_apical_negg": [0.1595289707183838, 0.19067716598510742, 0.4023829698562622], "cos_Ac_negg": [0.32851630449295044, 0.4066682457923889, 0.716888427734375], "r_norm": [1.0504348278045654, 1.0283509492874146, 1.0469398498535156]}, {"step": 400, "epoch": 0, "train_loss": 0.2309410572052002, "cos_r_negg": [0.13200967013835907, 0.22264397144317627, 0.35186904668807983], "cos_innovation_negg": [0.34929394721984863, 0.4233340620994568, 0.7414632439613342], "cos_apical_negg": [0.13200968503952026, 0.22264397144317627, 0.3518690764904022], "cos_Ac_negg": [0.3492230772972107, 0.4233376383781433, 0.7414527535438538], "r_norm": [0.9227346777915955, 0.9097601175308228, 0.9216660857200623]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9125, "test_loss": 0.34449742889404295}, {"step": 500, "epoch": 1, "train_loss": 0.3173186480998993, "cos_r_negg": [0.13482621312141418, 0.16621845960617065, 0.4622615575790405], "cos_innovation_negg": [0.342474102973938, 0.41030460596084595, 0.7856221199035645], "cos_apical_negg": [0.13482621312141418, 0.16621847450733185, 0.4622615575790405], "cos_Ac_negg": [0.3423904776573181, 0.4102475047111511, 0.7854601144790649], "r_norm": [0.8716844916343689, 0.8534141778945923, 0.8655869960784912], "ldr": {"dL_sdil": -0.03218638896942139, "dL_bp": -0.042296916246414185, "ratio": 0.7609630163558333}}, {"step": 600, "epoch": 1, "train_loss": 0.19914954900741577, "cos_r_negg": [0.10872647911310196, 0.15109533071517944, 0.38549095392227173], "cos_innovation_negg": [0.3493943512439728, 0.3854026794433594, 0.7403441071510315], "cos_apical_negg": [0.10872647166252136, 0.15109533071517944, 0.38549095392227173], "cos_Ac_negg": [0.3493119478225708, 0.38537198305130005, 0.7402387857437134], "r_norm": [0.7328153848648071, 0.725242555141449, 0.7366950511932373]}, {"step": 700, "epoch": 1, "train_loss": 0.26646754145622253, "cos_r_negg": [0.11888409405946732, 0.15967610478401184, 0.45279645919799805], "cos_innovation_negg": [0.32481685280799866, 0.4027654826641083, 0.7821754217147827], "cos_apical_negg": [0.11888409405946732, 0.15967610478401184, 0.45279648900032043], "cos_Ac_negg": [0.32472869753837585, 0.4027591347694397, 0.782071590423584], "r_norm": [0.8986352682113647, 0.8939275741577148, 0.9049227833747864]}, {"step": 800, "epoch": 1, "train_loss": 0.33199891448020935, "cos_r_negg": [0.08395060896873474, 0.11295019090175629, 0.07632268965244293], "cos_innovation_negg": [0.34351646900177, 0.3710211515426636, 0.6842052936553955], "cos_apical_negg": [0.08395060896873474, 0.11295019090175629, 0.07632268965244293], "cos_Ac_negg": [0.3434521555900574, 0.3709302246570587, 0.6840815544128418], "r_norm": [0.7469301223754883, 0.7383431196212769, 0.753665030002594]}, {"step": 900, "epoch": 1, "train_loss": 0.2762279808521271, "cos_r_negg": [0.12050512433052063, 0.13892970979213715, 0.03243357688188553], "cos_innovation_negg": [0.3857177495956421, 0.4192531108856201, 0.6353157162666321], "cos_apical_negg": [0.12050511687994003, 0.13892970979213715, 0.03243357688188553], "cos_Ac_negg": [0.38564231991767883, 0.4192121624946594, 0.6353316903114319], "r_norm": [0.7485752701759338, 0.7508242130279541, 0.760019838809967]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9348, "test_loss": 0.2599320869445801}, {"step": 1000, "epoch": 2, "train_loss": 0.14405804872512817, "cos_r_negg": [0.11138047277927399, 0.12647497653961182, 0.06470959633588791], "cos_innovation_negg": [0.3511101007461548, 0.3959304094314575, 0.6778622269630432], "cos_apical_negg": [0.11138047277927399, 0.12647497653961182, 0.06470959633588791], "cos_Ac_negg": [0.3510010838508606, 0.39592984318733215, 0.6778634786605835], "r_norm": [0.6931792497634888, 0.6907463073730469, 0.6990920305252075], "ldr": {"dL_sdil": -0.02318418025970459, "dL_bp": -0.031096190214157104, "ratio": 0.7455633664457575}}, {"step": 1100, "epoch": 2, "train_loss": 0.34493690729141235, "cos_r_negg": [0.08756481856107712, 0.11450334638357162, 0.29389017820358276], "cos_innovation_negg": [0.36921244859695435, 0.4279465973377228, 0.769111156463623], "cos_apical_negg": [0.08756481856107712, 0.11450335383415222, 0.29389017820358276], "cos_Ac_negg": [0.3691472113132477, 0.42785149812698364, 0.7690554857254028], "r_norm": [0.6190230846405029, 0.617702841758728, 0.6250365972518921]}, {"step": 1200, "epoch": 2, "train_loss": 0.29518693685531616, "cos_r_negg": [0.06379122287034988, 0.12758740782737732, 0.22313383221626282], "cos_innovation_negg": [0.378059983253479, 0.46550843119621277, 0.7720991373062134], "cos_apical_negg": [0.06379122287034988, 0.12758740782737732, 0.22313383221626282], "cos_Ac_negg": [0.37803614139556885, 0.46550166606903076, 0.7721033096313477], "r_norm": [0.5919125080108643, 0.5898166298866272, 0.5995679497718811]}, {"step": 1300, "epoch": 2, "train_loss": 0.274957537651062, "cos_r_negg": [0.08221554756164551, 0.10883848369121552, 0.296045184135437], "cos_innovation_negg": [0.3714882731437683, 0.4087797999382019, 0.6893191933631897], "cos_apical_negg": [0.08221554756164551, 0.10883848369121552, 0.296045184135437], "cos_Ac_negg": [0.3714684545993805, 0.4087407886981964, 0.6893448233604431], "r_norm": [0.7115012407302856, 0.7127417325973511, 0.7193174362182617]}, {"step": 1400, "epoch": 2, "train_loss": 0.26490557193756104, "cos_r_negg": [0.07762452960014343, 0.05591243505477905, 0.3536660075187683], "cos_innovation_negg": [0.37052151560783386, 0.43136483430862427, 0.7564950585365295], "cos_apical_negg": [0.07762452960014343, 0.05591243505477905, 0.3536660075187683], "cos_Ac_negg": [0.3696705400943756, 0.43082448840141296, 0.7561050653457642], "r_norm": [0.5983796715736389, 0.5932151675224304, 0.6052750945091248]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9375, "test_loss": 0.24186600036621095}, {"step": 1500, "epoch": 3, "train_loss": 0.28851747512817383, "cos_r_negg": [0.09779243171215057, 0.06952666491270065, 0.3150227963924408], "cos_innovation_negg": [0.37762948870658875, 0.4491156041622162, 0.7497493028640747], "cos_apical_negg": [0.09779242426156998, 0.06952666491270065, 0.3150227665901184], "cos_Ac_negg": [0.376889705657959, 0.4486525058746338, 0.7494707107543945], "r_norm": [0.5530264377593994, 0.5480079650878906, 0.5555835366249084], "ldr": {"dL_sdil": -0.014817297458648682, "dL_bp": -0.03878897428512573, "ratio": 0.38199766123567275}}, {"step": 1600, "epoch": 3, "train_loss": 0.3174806237220764, "cos_r_negg": [0.06319668143987656, 0.08834267407655716, 0.22097212076187134], "cos_innovation_negg": [0.3427238464355469, 0.4141557216644287, 0.7424756288528442], "cos_apical_negg": [0.06319668143987656, 0.08834266662597656, 0.22097212076187134], "cos_Ac_negg": [0.3426951766014099, 0.41411417722702026, 0.7424772381782532], "r_norm": [0.5841406583786011, 0.5867191553115845, 0.5884642601013184]}, {"step": 1700, "epoch": 3, "train_loss": 0.10985104739665985, "cos_r_negg": [0.08890071511268616, 0.10797759890556335, 0.3917781114578247], "cos_innovation_negg": [0.3356766104698181, 0.4258178472518921, 0.744598388671875], "cos_apical_negg": [0.08890072256326675, 0.10797760635614395, 0.3917781114578247], "cos_Ac_negg": [0.3357349634170532, 0.42589181661605835, 0.7445431351661682], "r_norm": [0.6230787634849548, 0.6204631328582764, 0.6286985874176025]}, {"step": 1800, "epoch": 3, "train_loss": 0.06924162060022354, "cos_r_negg": [0.11762216687202454, 0.12214531004428864, 0.47433561086654663], "cos_innovation_negg": [0.39117446541786194, 0.48938852548599243, 0.7135990262031555], "cos_apical_negg": [0.11762216687202454, 0.12214531004428864, 0.47433561086654663], "cos_Ac_negg": [0.3910622000694275, 0.4894335865974426, 0.7133069038391113], "r_norm": [0.7417702078819275, 0.7404168844223022, 0.7431856393814087]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9385, "test_loss": 0.25450943908691404}, {"step": 1900, "epoch": 4, "train_loss": 0.17704644799232483, "cos_r_negg": [0.08445397019386292, 0.08889074623584747, 0.3843984305858612], "cos_innovation_negg": [0.35447216033935547, 0.4475480914115906, 0.7383006811141968], "cos_apical_negg": [0.08445397019386292, 0.08889074623584747, 0.3843984603881836], "cos_Ac_negg": [0.3544359803199768, 0.4474848508834839, 0.7381622791290283], "r_norm": [0.6404048800468445, 0.6412316560745239, 0.6484839916229248]}, {"step": 2000, "epoch": 4, "train_loss": 0.21606126427650452, "cos_r_negg": [0.07945279777050018, 0.07109078019857407, 0.4097805917263031], "cos_innovation_negg": [0.33481740951538086, 0.4295274317264557, 0.7135540246963501], "cos_apical_negg": [0.07945279777050018, 0.07109078019857407, 0.4097805917263031], "cos_Ac_negg": [0.3350074887275696, 0.4299665093421936, 0.7135474681854248], "r_norm": [0.5689743757247925, 0.5581024885177612, 0.5706900358200073], "ldr": {"dL_sdil": -0.038610026240348816, "dL_bp": -0.08081431686878204, "ratio": 0.47776220521717455}}, {"step": 2100, "epoch": 4, "train_loss": 0.18260759115219116, "cos_r_negg": [0.07964015007019043, 0.09623073041439056, 0.34003013372421265], "cos_innovation_negg": [0.3523395359516144, 0.43928754329681396, 0.7320300340652466], "cos_apical_negg": [0.07964015007019043, 0.09623073041439056, 0.34003013372421265], "cos_Ac_negg": [0.3522557020187378, 0.4392588138580322, 0.7317688465118408], "r_norm": [0.5521249771118164, 0.544990062713623, 0.551296591758728]}, {"step": 2200, "epoch": 4, "train_loss": 0.11431927233934402, "cos_r_negg": [0.058552246540784836, 0.07483963668346405, 0.2631024122238159], "cos_innovation_negg": [0.3422366976737976, 0.45720887184143066, 0.7212303876876831], "cos_apical_negg": [0.05855224281549454, 0.07483963668346405, 0.2631024122238159], "cos_Ac_negg": [0.3421591818332672, 0.4572329521179199, 0.7209049463272095], "r_norm": [0.4327749013900757, 0.431601345539093, 0.4365268349647522]}, {"step": 2300, "epoch": 4, "train_loss": 0.3545019328594208, "cos_r_negg": [0.05145160108804703, 0.10118147730827332, 0.2988181412220001], "cos_innovation_negg": [0.37262725830078125, 0.5018264651298523, 0.7596726417541504], "cos_apical_negg": [0.05145159736275673, 0.10118147730827332, 0.2988181412220001], "cos_Ac_negg": [0.3725670576095581, 0.5017696022987366, 0.7596282362937927], "r_norm": [0.44180363416671753, 0.44251295924186707, 0.4432508945465088]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9426, "test_loss": 0.24033984069824219}, {"step": 2400, "epoch": 5, "train_loss": 0.13781969249248505, "cos_r_negg": [0.08198727667331696, 0.10721886157989502, 0.3741753101348877], "cos_innovation_negg": [0.32792022824287415, 0.4357667565345764, 0.8825811743736267], "cos_apical_negg": [0.08198726922273636, 0.10721886157989502, 0.3741753101348877], "cos_Ac_negg": [0.3278215527534485, 0.43569254875183105, 0.882398247718811], "r_norm": [0.568074107170105, 0.5682215690612793, 0.5739967823028564]}, {"step": 2500, "epoch": 5, "train_loss": 0.2105558216571808, "cos_r_negg": [0.07025723904371262, 0.05517081916332245, 0.37762144207954407], "cos_innovation_negg": [0.3441767692565918, 0.4478762149810791, 0.7895771265029907], "cos_apical_negg": [0.07025723904371262, 0.05517082288861275, 0.37762144207954407], "cos_Ac_negg": [0.34406840801239014, 0.4480290412902832, 0.7895047664642334], "r_norm": [0.4344387650489807, 0.429985910654068, 0.4351077973842621], "ldr": {"dL_sdil": -0.019099697470664978, "dL_bp": -0.04359477758407593, "ratio": 0.4381189337147029}}, {"step": 2600, "epoch": 5, "train_loss": 0.14259913563728333, "cos_r_negg": [0.06413869559764862, 0.08747546374797821, 0.3348381817340851], "cos_innovation_negg": [0.34231239557266235, 0.4611664414405823, 0.8132108449935913], "cos_apical_negg": [0.06413869559764862, 0.08747545629739761, 0.3348381817340851], "cos_Ac_negg": [0.3423386216163635, 0.4612843990325928, 0.8129971027374268], "r_norm": [0.46504828333854675, 0.4645615816116333, 0.46562665700912476]}, {"step": 2700, "epoch": 5, "train_loss": 0.14656184613704681, "cos_r_negg": [0.06414548307657242, 0.05336803197860718, 0.21878519654273987], "cos_innovation_negg": [0.3596385717391968, 0.41244661808013916, 0.7379372715950012], "cos_apical_negg": [0.06414547562599182, 0.05336803197860718, 0.21878519654273987], "cos_Ac_negg": [0.3595573902130127, 0.41238903999328613, 0.7376086711883545], "r_norm": [0.45414695143699646, 0.4527623951435089, 0.45553410053253174]}, {"step": 2800, "epoch": 5, "train_loss": 0.2994046211242676, "cos_r_negg": [0.05465333163738251, 0.07265643030405045, 0.2811175286769867], "cos_innovation_negg": [0.3468356132507324, 0.4211798310279846, 0.7703515887260437], "cos_apical_negg": [0.05465332791209221, 0.07265643775463104, 0.2811175286769867], "cos_Ac_negg": [0.3464857339859009, 0.42129141092300415, 0.7699546813964844], "r_norm": [0.449910968542099, 0.44745439291000366, 0.4500837028026581]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9482, "test_loss": 0.21917556610107422}, {"step": 2900, "epoch": 6, "train_loss": 0.13684412837028503, "cos_r_negg": [0.06713424623012543, 0.030654802918434143, 0.30837690830230713], "cos_innovation_negg": [0.39000511169433594, 0.4487501084804535, 0.8027445673942566], "cos_apical_negg": [0.06713424623012543, 0.03065480850636959, 0.30837690830230713], "cos_Ac_negg": [0.3899167776107788, 0.4486718773841858, 0.80238276720047], "r_norm": [0.4206772446632385, 0.42287734150886536, 0.42088812589645386]}, {"step": 3000, "epoch": 6, "train_loss": 0.32784920930862427, "cos_r_negg": [0.09284724295139313, 0.09827914834022522, 0.3178602457046509], "cos_innovation_negg": [0.3673955202102661, 0.49373292922973633, 0.7591451406478882], "cos_apical_negg": [0.09284724295139313, 0.09827914834022522, 0.3178602457046509], "cos_Ac_negg": [0.3669975697994232, 0.4936269223690033, 0.7590146064758301], "r_norm": [0.5538517832756042, 0.5461391806602478, 0.5484142303466797], "ldr": {"dL_sdil": -0.041023969650268555, "dL_bp": -0.07921682298183441, "ratio": 0.5178694134158341}}, {"step": 3100, "epoch": 6, "train_loss": 0.14292265474796295, "cos_r_negg": [0.06172025203704834, 0.05513174086809158, 0.3732379674911499], "cos_innovation_negg": [0.3735494613647461, 0.4711996912956238, 0.8251690864562988], "cos_apical_negg": [0.06172025203704834, 0.05513174831867218, 0.3732379674911499], "cos_Ac_negg": [0.3735364079475403, 0.471378356218338, 0.8250607848167419], "r_norm": [0.5026163458824158, 0.509120523929596, 0.5035586357116699]}, {"step": 3200, "epoch": 6, "train_loss": 0.1502373367547989, "cos_r_negg": [0.06354902684688568, 0.06215986981987953, 0.26972097158432007], "cos_innovation_negg": [0.31604406237602234, 0.3920234739780426, 0.8084297180175781], "cos_apical_negg": [0.06354902684688568, 0.06215986609458923, 0.26972097158432007], "cos_Ac_negg": [0.31607282161712646, 0.3919937014579773, 0.8079515695571899], "r_norm": [0.4039853811264038, 0.403599351644516, 0.4045742154121399]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9535, "test_loss": 0.2096079750061035}, {"step": 3300, "epoch": 7, "train_loss": 0.1335640698671341, "cos_r_negg": [0.06980779767036438, 0.09236127883195877, 0.15082693099975586], "cos_innovation_negg": [0.3417547345161438, 0.4821212589740753, 0.7918124794960022], "cos_apical_negg": [0.06980779767036438, 0.09236127883195877, 0.15082691609859467], "cos_Ac_negg": [0.34179019927978516, 0.4822649359703064, 0.791712760925293], "r_norm": [0.5176693201065063, 0.5247974395751953, 0.5229960680007935]}, {"step": 3400, "epoch": 7, "train_loss": 0.19798503816127777, "cos_r_negg": [0.07338063418865204, 0.06492847204208374, 0.113883838057518], "cos_innovation_negg": [0.3281344175338745, 0.4483921527862549, 0.8133120536804199], "cos_apical_negg": [0.07338063418865204, 0.06492847204208374, 0.113883838057518], "cos_Ac_negg": [0.3280527591705322, 0.4484575390815735, 0.8128377795219421], "r_norm": [0.4960020184516907, 0.4983512759208679, 0.4969604015350342]}, {"step": 3500, "epoch": 7, "train_loss": 0.15915080904960632, "cos_r_negg": [0.0944727435708046, 0.026904992759227753, 0.18848344683647156], "cos_innovation_negg": [0.3491503596305847, 0.44440147280693054, 0.7386237382888794], "cos_apical_negg": [0.0944727435708046, 0.026904992759227753, 0.18848344683647156], "cos_Ac_negg": [0.34957247972488403, 0.4445277154445648, 0.7377989888191223], "r_norm": [0.46674031019210815, 0.47091615200042725, 0.4683659076690674], "ldr": {"dL_sdil": -0.021384015679359436, "dL_bp": -0.0503864586353302, "ratio": 0.42440005228637556}}, {"step": 3600, "epoch": 7, "train_loss": 0.04387524724006653, "cos_r_negg": [0.046654507517814636, 0.010634372942149639, 0.11703355610370636], "cos_innovation_negg": [0.3120917081832886, 0.39470893144607544, 0.7540159225463867], "cos_apical_negg": [0.046654507517814636, 0.010634373873472214, 0.11703355610370636], "cos_Ac_negg": [0.3120144009590149, 0.3947693109512329, 0.75221848487854], "r_norm": [0.37566274404525757, 0.3760484457015991, 0.3757573366165161]}, {"step": 3700, "epoch": 7, "train_loss": 0.058575041592121124, "cos_r_negg": [0.04966064542531967, 0.04389099031686783, 0.22908291220664978], "cos_innovation_negg": [0.33947324752807617, 0.4016231298446655, 0.8086131811141968], "cos_apical_negg": [0.04966064915060997, 0.04389099404215813, 0.22908291220664978], "cos_Ac_negg": [0.3395770490169525, 0.4018130600452423, 0.8080916404724121], "r_norm": [0.3791559040546417, 0.37757930159568787, 0.38096868991851807]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9456, "test_loss": 0.24103100204467773}, {"step": 3800, "epoch": 8, "train_loss": 0.19025324285030365, "cos_r_negg": [0.08102741092443466, 0.01857561618089676, 0.3476852476596832], "cos_innovation_negg": [0.3338545858860016, 0.43275606632232666, 0.820645809173584], "cos_apical_negg": [0.08102741092443466, 0.01857561245560646, 0.34768521785736084], "cos_Ac_negg": [0.33342835307121277, 0.4327048361301422, 0.8198139667510986], "r_norm": [0.39218252897262573, 0.3957356810569763, 0.3993185758590698]}, {"step": 3900, "epoch": 8, "train_loss": 0.34581154584884644, "cos_r_negg": [0.054648131132125854, 0.05118422955274582, 0.26381927728652954], "cos_innovation_negg": [0.3540130853652954, 0.4399789571762085, 0.9029856324195862], "cos_apical_negg": [0.054648131132125854, 0.05118422955274582, 0.26381930708885193], "cos_Ac_negg": [0.3538229465484619, 0.44010695815086365, 0.9017641544342041], "r_norm": [0.4435705840587616, 0.4424494504928589, 0.4460024833679199]}, {"step": 4000, "epoch": 8, "train_loss": 0.11512741446495056, "cos_r_negg": [0.06321237981319427, 0.06220477819442749, 0.2510380744934082], "cos_innovation_negg": [0.360689640045166, 0.4856993854045868, 0.8238543272018433], "cos_apical_negg": [0.06321237981319427, 0.06220477446913719, 0.2510380744934082], "cos_Ac_negg": [0.3610338270664215, 0.4859244227409363, 0.8238083720207214], "r_norm": [0.3842754364013672, 0.3864317834377289, 0.3872900605201721], "ldr": {"dL_sdil": -0.03004211187362671, "dL_bp": -0.07631881535053253, "ratio": 0.39363965144955676}}, {"step": 4100, "epoch": 8, "train_loss": 0.0921652764081955, "cos_r_negg": [0.06788378953933716, 0.05737902969121933, 0.19586181640625], "cos_innovation_negg": [0.35160183906555176, 0.4368787109851837, 0.7348260879516602], "cos_apical_negg": [0.06788378953933716, 0.05737902969121933, 0.1958618015050888], "cos_Ac_negg": [0.35199347138404846, 0.43733251094818115, 0.7354164123535156], "r_norm": [0.4068611264228821, 0.4173135757446289, 0.4115045964717865]}, {"step": 4200, "epoch": 8, "train_loss": 0.21099771559238434, "cos_r_negg": [0.03902527317404747, 0.05180132016539574, 0.19548070430755615], "cos_innovation_negg": [0.38638246059417725, 0.4414784610271454, 0.7923601865768433], "cos_apical_negg": [0.03902527689933777, 0.05180132016539574, 0.19548071920871735], "cos_Ac_negg": [0.38578999042510986, 0.4416012167930603, 0.7907538414001465], "r_norm": [0.39080530405044556, 0.38810592889785767, 0.391198992729187]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9437, "test_loss": 0.24913544845581054}, {"step": 4300, "epoch": 9, "train_loss": 0.10257379710674286, "cos_r_negg": [0.0578710101544857, 0.06124154105782509, 0.16410021483898163], "cos_innovation_negg": [0.3881415128707886, 0.47438478469848633, 0.8227272033691406], "cos_apical_negg": [0.0578710101544857, 0.06124153360724449, 0.16410021483898163], "cos_Ac_negg": [0.3880169093608856, 0.47455883026123047, 0.8224486708641052], "r_norm": [0.43919825553894043, 0.43225860595703125, 0.4337231516838074]}, {"step": 4400, "epoch": 9, "train_loss": 0.10444098711013794, "cos_r_negg": [0.03795100003480911, 0.03195874020457268, 0.06112508475780487], "cos_innovation_negg": [0.3105248808860779, 0.38150686025619507, 0.8311172723770142], "cos_apical_negg": [0.03795100003480911, 0.03195874020457268, 0.06112508103251457], "cos_Ac_negg": [0.31050512194633484, 0.3816024959087372, 0.8309045433998108], "r_norm": [0.3962266445159912, 0.39936453104019165, 0.395593523979187]}, {"step": 4500, "epoch": 9, "train_loss": 0.2180986851453781, "cos_r_negg": [0.035004157572984695, 0.05222100764513016, 0.005726968869566917], "cos_innovation_negg": [0.3067225217819214, 0.40333491563796997, 0.8238222599029541], "cos_apical_negg": [0.0350041538476944, 0.052221011370420456, 0.005726972594857216], "cos_Ac_negg": [0.30659380555152893, 0.403276264667511, 0.8236179351806641], "r_norm": [0.32833749055862427, 0.3300885558128357, 0.3319564461708069], "ldr": {"dL_sdil": -0.007901772856712341, "dL_bp": -0.02477891743183136, "ratio": 0.31889096359640023}}, {"step": 4600, "epoch": 9, "train_loss": 0.09447373449802399, "cos_r_negg": [0.048628177493810654, 0.010213809087872505, 0.12252732366323471], "cos_innovation_negg": [0.32184451818466187, 0.4015880525112152, 0.7796953320503235], "cos_apical_negg": [0.048628177493810654, 0.010213812813162804, 0.12252731621265411], "cos_Ac_negg": [0.32202592492103577, 0.4013347625732422, 0.7796410322189331], "r_norm": [0.3619610667228699, 0.3635615408420563, 0.3631513714790344]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.957, "test_loss": 0.20237664337158204}, {"step": 4700, "epoch": 10, "train_loss": 0.12431761622428894, "cos_r_negg": [0.05357087776064873, 0.04383087903261185, 0.1457587331533432], "cos_innovation_negg": [0.3576279878616333, 0.4251352548599243, 0.831009030342102], "cos_apical_negg": [0.05357087403535843, 0.04383087903261185, 0.1457587331533432], "cos_Ac_negg": [0.3574097156524658, 0.4258958697319031, 0.8296326398849487], "r_norm": [0.29066699743270874, 0.2902194857597351, 0.2891004681587219]}, {"step": 4800, "epoch": 10, "train_loss": 0.16371572017669678, "cos_r_negg": [0.029036926105618477, 0.004269483499228954, 0.19963490962982178], "cos_innovation_negg": [0.32341933250427246, 0.4272913336753845, 0.8171665668487549], "cos_apical_negg": [0.029036924242973328, 0.004269478842616081, 0.19963490962982178], "cos_Ac_negg": [0.3231048882007599, 0.42749524116516113, 0.813264012336731], "r_norm": [0.285106360912323, 0.28631317615509033, 0.2873285114765167]}, {"step": 4900, "epoch": 10, "train_loss": 0.14902347326278687, "cos_r_negg": [0.04620438814163208, 0.02644183300435543, 0.15327578783035278], "cos_innovation_negg": [0.3307992219924927, 0.44949227571487427, 0.8080215454101562], "cos_apical_negg": [0.04620438814163208, 0.026441829279065132, 0.15327578783035278], "cos_Ac_negg": [0.3309200704097748, 0.4503331780433655, 0.8067384362220764], "r_norm": [0.3732837438583374, 0.3736773729324341, 0.37527549266815186]}, {"step": 5000, "epoch": 10, "train_loss": 0.09991974383592606, "cos_r_negg": [0.049699798226356506, 0.08670321106910706, 0.07492829859256744], "cos_innovation_negg": [0.3452371656894684, 0.4075286090373993, 0.813042402267456], "cos_apical_negg": [0.049699798226356506, 0.08670320361852646, 0.07492829859256744], "cos_Ac_negg": [0.34534674882888794, 0.40769147872924805, 0.8129086494445801], "r_norm": [0.3537364900112152, 0.3528369069099426, 0.3551481366157532], "ldr": {"dL_sdil": -0.017207548022270203, "dL_bp": -0.04944072663784027, "ratio": 0.3480439951523715}}, {"step": 5100, "epoch": 10, "train_loss": 0.12895873188972473, "cos_r_negg": [0.04770016297698021, 0.03193221986293793, 0.21643300354480743], "cos_innovation_negg": [0.3295574486255646, 0.42878806591033936, 0.8419816493988037], "cos_apical_negg": [0.04770016297698021, 0.031932227313518524, 0.21643300354480743], "cos_Ac_negg": [0.3295457363128662, 0.42891111969947815, 0.8411244750022888], "r_norm": [0.3373553156852722, 0.33803504705429077, 0.3393421769142151]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.951, "test_loss": 0.23327355194091798}, {"step": 5200, "epoch": 11, "train_loss": 0.08230389654636383, "cos_r_negg": [0.03477941453456879, 0.005066273733973503, 0.15518972277641296], "cos_innovation_negg": [0.310738742351532, 0.4165608882904053, 0.8202950954437256], "cos_apical_negg": [0.034779418259859085, 0.005066270008683205, 0.15518973767757416], "cos_Ac_negg": [0.31071263551712036, 0.4168030321598053, 0.8189512491226196], "r_norm": [0.3066258430480957, 0.30801162123680115, 0.3093065619468689]}, {"step": 5300, "epoch": 11, "train_loss": 0.14475546777248383, "cos_r_negg": [0.03744753822684288, 0.0545089952647686, 0.29369843006134033], "cos_innovation_negg": [0.3429303467273712, 0.4583066403865814, 0.8737197518348694], "cos_apical_negg": [0.03744753450155258, 0.054508984088897705, 0.29369840025901794], "cos_Ac_negg": [0.342940092086792, 0.4582170844078064, 0.8718173503875732], "r_norm": [0.35003840923309326, 0.35217857360839844, 0.3526148796081543]}, {"step": 5400, "epoch": 11, "train_loss": 0.2559114694595337, "cos_r_negg": [0.02582029439508915, 0.04114362597465515, 0.13393956422805786], "cos_innovation_negg": [0.30870330333709717, 0.4293370544910431, 0.8331000804901123], "cos_apical_negg": [0.025820299983024597, 0.04114362597465515, 0.13393956422805786], "cos_Ac_negg": [0.30875164270401, 0.429548978805542, 0.8332539796829224], "r_norm": [0.3590816557407379, 0.35885876417160034, 0.3555552363395691]}, {"step": 5500, "epoch": 11, "train_loss": 0.1681821644306183, "cos_r_negg": [0.04578680545091629, 0.05125431716442108, 0.0690905898809433], "cos_innovation_negg": [0.2915537357330322, 0.41783082485198975, 0.7715979218482971], "cos_apical_negg": [0.04578680545091629, 0.05125431716442108, 0.0690905898809433], "cos_Ac_negg": [0.29153886437416077, 0.41815367341041565, 0.7711772918701172], "r_norm": [0.3617756962776184, 0.36520957946777344, 0.35974299907684326], "ldr": {"dL_sdil": -0.015750110149383545, "dL_bp": -0.03815075755119324, "ratio": 0.41283872615763906}}, {"step": 5600, "epoch": 11, "train_loss": 0.10013377666473389, "cos_r_negg": [0.04685045778751373, 0.05785603076219559, 0.2090340405702591], "cos_innovation_negg": [0.35300010442733765, 0.4542841911315918, 0.7812929153442383], "cos_apical_negg": [0.04685045778751373, 0.05785602703690529, 0.2090340405702591], "cos_Ac_negg": [0.35341233015060425, 0.4559386968612671, 0.7789859771728516], "r_norm": [0.4355519711971283, 0.43699824810028076, 0.4287186861038208]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9507, "test_loss": 0.23671178817749022}, {"step": 5700, "epoch": 12, "train_loss": 0.036010392010211945, "cos_r_negg": [0.007591079920530319, -0.0011415127664804459, 0.048039890825748444], "cos_innovation_negg": [0.273660272359848, 0.4013635516166687, 0.8559930324554443], "cos_apical_negg": [0.007591081317514181, -0.0011415127664804459, 0.048039887100458145], "cos_Ac_negg": [0.2741205394268036, 0.4025787115097046, 0.855581521987915], "r_norm": [0.26295018196105957, 0.2653757929801941, 0.2609485685825348]}, {"step": 5800, "epoch": 12, "train_loss": 0.06087285280227661, "cos_r_negg": [0.030401382595300674, 0.002562267705798149, 0.24839895963668823], "cos_innovation_negg": [0.326499342918396, 0.4764612913131714, 0.8363528251647949], "cos_apical_negg": [0.030401380732655525, 0.002562267705798149, 0.24839895963668823], "cos_Ac_negg": [0.32592064142227173, 0.4774336516857147, 0.8326979875564575], "r_norm": [0.35251328349113464, 0.3532651960849762, 0.352918803691864]}, {"step": 5900, "epoch": 12, "train_loss": 0.0821932852268219, "cos_r_negg": [0.021442102268338203, 0.015266858972609043, 0.20375409722328186], "cos_innovation_negg": [0.3404744863510132, 0.4297502636909485, 0.763418436050415], "cos_apical_negg": [0.021442092955112457, 0.015266858041286469, 0.20375409722328186], "cos_Ac_negg": [0.34027743339538574, 0.4302564263343811, 0.7633066773414612], "r_norm": [0.3595448434352875, 0.3570975065231323, 0.3590535521507263]}, {"step": 6000, "epoch": 12, "train_loss": 0.008466094732284546, "cos_r_negg": [0.025640254840254784, 0.03812666982412338, 0.1619781255722046], "cos_innovation_negg": [0.34594208002090454, 0.43582791090011597, 0.7938475608825684], "cos_apical_negg": [0.025640252977609634, 0.038126666098833084, 0.1619781255722046], "cos_Ac_negg": [0.3454383611679077, 0.43601155281066895, 0.7889047861099243], "r_norm": [0.31336599588394165, 0.3176407814025879, 0.3096732497215271], "ldr": {"dL_sdil": -0.018425673246383667, "dL_bp": -0.049481213092803955, "ratio": 0.37237715275544264}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9525, "test_loss": 0.2193112091064453}, {"step": 6100, "epoch": 13, "train_loss": 0.11159200966358185, "cos_r_negg": [0.0439818799495697, 0.032109104096889496, 0.18376143276691437], "cos_innovation_negg": [0.32291585206985474, 0.48050880432128906, 0.8388372659683228], "cos_apical_negg": [0.0439818799495697, 0.032109104096889496, 0.18376144766807556], "cos_Ac_negg": [0.32420697808265686, 0.48160845041275024, 0.8389242887496948], "r_norm": [0.39198821783065796, 0.3899760842323303, 0.38981926441192627]}, {"step": 6200, "epoch": 13, "train_loss": 0.13688595592975616, "cos_r_negg": [0.03425388038158417, 0.0277108121663332, 0.1322983354330063], "cos_innovation_negg": [0.3180942237377167, 0.4551396667957306, 0.7792147397994995], "cos_apical_negg": [0.034253884106874466, 0.02771081030368805, 0.1322983354330063], "cos_Ac_negg": [0.31836777925491333, 0.4565110206604004, 0.7771570682525635], "r_norm": [0.3241232931613922, 0.3264669179916382, 0.32752513885498047]}, {"step": 6300, "epoch": 13, "train_loss": 0.0561244897544384, "cos_r_negg": [0.0288222748786211, 0.032034892588853836, 0.15862321853637695], "cos_innovation_negg": [0.2943158745765686, 0.3951963782310486, 0.8620610237121582], "cos_apical_negg": [0.02882227674126625, 0.032034896314144135, 0.15862321853637695], "cos_Ac_negg": [0.2938181161880493, 0.3969143331050873, 0.8599331974983215], "r_norm": [0.320553183555603, 0.3219207227230072, 0.3195515275001526]}, {"step": 6400, "epoch": 13, "train_loss": 0.04703892767429352, "cos_r_negg": [0.03496722877025604, 0.030867431312799454, 0.18513604998588562], "cos_innovation_negg": [0.3197341859340668, 0.43913203477859497, 0.8335174918174744], "cos_apical_negg": [0.03496722877025604, 0.030867429450154305, 0.18513604998588562], "cos_Ac_negg": [0.321494996547699, 0.44213366508483887, 0.8298313617706299], "r_norm": [0.3697455823421478, 0.3741699457168579, 0.36658036708831787]}, {"step": 6500, "epoch": 13, "train_loss": 0.016488533467054367, "cos_r_negg": [0.052222929894924164, 0.06285339593887329, 0.13343417644500732], "cos_innovation_negg": [0.38896992802619934, 0.49758970737457275, 0.8522046804428101], "cos_apical_negg": [0.052222929894924164, 0.06285339593887329, 0.13343417644500732], "cos_Ac_negg": [0.3891015648841858, 0.4979743957519531, 0.8522868156433105], "r_norm": [0.3370938301086426, 0.3465665280818939, 0.3368062376976013], "ldr": {"dL_sdil": -0.024978965520858765, "dL_bp": -0.07083387672901154, "ratio": 0.35264151383977677}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9596, "test_loss": 0.1998619110107422}, {"step": 6600, "epoch": 14, "train_loss": 0.04038822650909424, "cos_r_negg": [0.026838652789592743, 0.051512375473976135, 0.08026068657636642], "cos_innovation_negg": [0.35957109928131104, 0.4205498993396759, 0.8914711475372314], "cos_apical_negg": [0.02683865651488304, 0.05151237174868584, 0.08026069402694702], "cos_Ac_negg": [0.3585135340690613, 0.4217907786369324, 0.8914788961410522], "r_norm": [0.31473827362060547, 0.3170691728591919, 0.3125041723251343]}, {"step": 6700, "epoch": 14, "train_loss": 0.02604137733578682, "cos_r_negg": [0.036674678325653076, 0.07491622865200043, 0.1431065797805786], "cos_innovation_negg": [0.3476559519767761, 0.40931612253189087, 0.8611284494400024], "cos_apical_negg": [0.036674678325653076, 0.07491622120141983, 0.1431065797805786], "cos_Ac_negg": [0.3479863405227661, 0.41027823090553284, 0.8615411520004272], "r_norm": [0.3823716938495636, 0.3841034770011902, 0.3771008849143982]}, {"step": 6800, "epoch": 14, "train_loss": 0.08187447488307953, "cos_r_negg": [0.03891880437731743, 0.027264870703220367, 0.03262750431895256], "cos_innovation_negg": [0.30911949276924133, 0.38232144713401794, 0.85636305809021], "cos_apical_negg": [0.03891880065202713, 0.02726486697793007, 0.032627515494823456], "cos_Ac_negg": [0.3091888427734375, 0.3834376633167267, 0.8598622679710388], "r_norm": [0.3174425959587097, 0.3199201822280884, 0.31509917974472046]}, {"step": 6900, "epoch": 14, "train_loss": 0.07461591809988022, "cos_r_negg": [0.044822920113801956, 0.058030031621456146, 0.08663206547498703], "cos_innovation_negg": [0.3657809793949127, 0.42271316051483154, 0.8233124017715454], "cos_apical_negg": [0.044822920113801956, 0.058030035346746445, 0.08663207292556763], "cos_Ac_negg": [0.36638933420181274, 0.42316633462905884, 0.8230905532836914], "r_norm": [0.3332439064979553, 0.3347223103046417, 0.33284109830856323]}, {"step": 7000, "epoch": 14, "train_loss": 0.06315820664167404, "cos_r_negg": [0.04397159069776535, 0.02858033776283264, 0.04650462418794632], "cos_innovation_negg": [0.3464168310165405, 0.4357408881187439, 0.8297408819198608], "cos_apical_negg": [0.04397159069776535, 0.028580334037542343, 0.046504613012075424], "cos_Ac_negg": [0.34667283296585083, 0.4369245767593384, 0.8286241292953491], "r_norm": [0.3260006308555603, 0.33002299070358276, 0.32008862495422363], "ldr": {"dL_sdil": -0.014171004295349121, "dL_bp": -0.04084871709346771, "ratio": 0.3469143048709176}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9588, "test_loss": 0.20080369186401367}], "final": {"test_acc": 0.9588, "test_loss": 0.20080369186401367, "wall_s": 25.22831654548645, "cos_r_negg": [0.03533933311700821, 0.05207996815443039, 0.037533730268478394], "cos_innovation_negg": [0.33997786045074463, 0.41935911774635315, 0.8430595993995667], "cos_apical_negg": [0.03533933684229851, 0.052079975605010986, 0.03753373771905899], "cos_Ac_negg": [0.34118103981018066, 0.4210580885410309, 0.8421542644500732]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_matched_s4.json b/results/nuis_ctrl_v1_rho0p2_matched_s4.json new file mode 100644 index 0000000..5126005 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_matched_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_matched_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.011682895943522453, 0.021248377859592438, 0.0025810941588133574], "cos_innovation_negg": [-0.010262858122587204, 0.024842172861099243, 0.006921787746250629], "cos_apical_negg": [-0.011682895943522453, 0.021248377859592438, 0.0025810941588133574], "cos_Ac_negg": [-0.010263703763484955, 0.02484174072742462, 0.006921331398189068], "r_norm": [4.861614227294922, 4.8644256591796875, 4.958981513977051], "ldr": {"dL_sdil": 0.009249448776245117, "dL_bp": -0.2576141357421875, "ratio": -0.03590427501036546}}, {"step": 100, "epoch": 0, "train_loss": 0.41053307056427, "cos_r_negg": [0.21120573580265045, 0.2628409266471863, 0.43200212717056274], "cos_innovation_negg": [0.41517841815948486, 0.4416387677192688, 0.7416012287139893], "cos_apical_negg": [0.21120572090148926, 0.2628409266471863, 0.43200209736824036], "cos_Ac_negg": [0.4149090051651001, 0.4414636194705963, 0.7415955662727356], "r_norm": [1.4688832759857178, 1.4861677885055542, 1.5087388753890991]}, {"step": 200, "epoch": 0, "train_loss": 0.6262565851211548, "cos_r_negg": [0.2109842300415039, 0.21873196959495544, 0.3458510637283325], "cos_innovation_negg": [0.3923499584197998, 0.3887983560562134, 0.730186939239502], "cos_apical_negg": [0.2109842300415039, 0.21873196959495544, 0.3458510637283325], "cos_Ac_negg": [0.39198464155197144, 0.38864755630493164, 0.7304191589355469], "r_norm": [1.3699445724487305, 1.3842023611068726, 1.4075835943222046]}, {"step": 300, "epoch": 0, "train_loss": 0.3782344460487366, "cos_r_negg": [0.16843444108963013, 0.20350274443626404, 0.3043299615383148], "cos_innovation_negg": [0.4008367359638214, 0.42071056365966797, 0.6820570230484009], "cos_apical_negg": [0.16843444108963013, 0.20350274443626404, 0.3043299615383148], "cos_Ac_negg": [0.40066319704055786, 0.42055952548980713, 0.6820881962776184], "r_norm": [0.988032877445221, 1.0014121532440186, 1.0196332931518555]}, {"step": 400, "epoch": 0, "train_loss": 0.28483903408050537, "cos_r_negg": [0.18906868994235992, 0.21057285368442535, 0.3197048604488373], "cos_innovation_negg": [0.4159192740917206, 0.42365968227386475, 0.8126862645149231], "cos_apical_negg": [0.18906867504119873, 0.21057286858558655, 0.3197048604488373], "cos_Ac_negg": [0.4157969355583191, 0.4235580563545227, 0.8126296997070312], "r_norm": [1.0828183889389038, 1.0980843305587769, 1.1216752529144287]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9122, "test_loss": 0.33212696533203123}, {"step": 500, "epoch": 1, "train_loss": 0.2540929913520813, "cos_r_negg": [0.15185591578483582, 0.15670140087604523, 0.26108109951019287], "cos_innovation_negg": [0.3839699923992157, 0.3891584575176239, 0.7882236838340759], "cos_apical_negg": [0.15185591578483582, 0.15670140087604523, 0.26108112931251526], "cos_Ac_negg": [0.3838613033294678, 0.3890269994735718, 0.7882048487663269], "r_norm": [0.8332318067550659, 0.8433594703674316, 0.8635990023612976], "ldr": {"dL_sdil": -0.01319509744644165, "dL_bp": -0.020045429468154907, "ratio": 0.6582596530248449}}, {"step": 600, "epoch": 1, "train_loss": 0.16460122168064117, "cos_r_negg": [0.15444540977478027, 0.15115073323249817, 0.33929991722106934], "cos_innovation_negg": [0.3953157663345337, 0.41563254594802856, 0.7793077826499939], "cos_apical_negg": [0.15444540977478027, 0.15115073323249817, 0.33929991722106934], "cos_Ac_negg": [0.39511990547180176, 0.4155126214027405, 0.7791947722434998], "r_norm": [0.9099339246749878, 0.9232620000839233, 0.9495981335639954]}, {"step": 700, "epoch": 1, "train_loss": 0.2918155789375305, "cos_r_negg": [0.15385684370994568, 0.16356827318668365, 0.29698213934898376], "cos_innovation_negg": [0.4167991280555725, 0.42821580171585083, 0.7697370648384094], "cos_apical_negg": [0.15385684370994568, 0.16356827318668365, 0.29698213934898376], "cos_Ac_negg": [0.4166952967643738, 0.42808353900909424, 0.7698770761489868], "r_norm": [0.9668311476707458, 0.9881653785705566, 1.0139029026031494]}, {"step": 800, "epoch": 1, "train_loss": 0.3391774594783783, "cos_r_negg": [0.14070671796798706, 0.154814213514328, 0.09407252073287964], "cos_innovation_negg": [0.3432130217552185, 0.39280059933662415, 0.7598002552986145], "cos_apical_negg": [0.14070671796798706, 0.154814213514328, 0.09407252073287964], "cos_Ac_negg": [0.34309136867523193, 0.39268428087234497, 0.7597470283508301], "r_norm": [0.9049474000930786, 0.9296468496322632, 0.9450775980949402]}, {"step": 900, "epoch": 1, "train_loss": 0.3571324050426483, "cos_r_negg": [0.11806076765060425, 0.13426010310649872, 0.1254441887140274], "cos_innovation_negg": [0.36654984951019287, 0.3533163070678711, 0.7541623115539551], "cos_apical_negg": [0.11806076765060425, 0.13426010310649872, 0.1254441738128662], "cos_Ac_negg": [0.36658594012260437, 0.35329481959342957, 0.7541159391403198], "r_norm": [0.7646726965904236, 0.7780411243438721, 0.8002845048904419]}, {"epoch_end": 1, "step": 938, "test_acc": 0.939, "test_loss": 0.23312706756591797}, {"step": 1000, "epoch": 2, "train_loss": 0.09978217631578445, "cos_r_negg": [0.13688361644744873, 0.15752843022346497, 0.19977553188800812], "cos_innovation_negg": [0.4013173580169678, 0.44258052110671997, 0.8329656720161438], "cos_apical_negg": [0.13688363134860992, 0.15752843022346497, 0.19977551698684692], "cos_Ac_negg": [0.40131014585494995, 0.44255679845809937, 0.8326481580734253], "r_norm": [0.7775167226791382, 0.7903605103492737, 0.8099894523620605], "ldr": {"dL_sdil": -0.030260175466537476, "dL_bp": -0.04607781767845154, "ratio": 0.6567189374658418}}, {"step": 1100, "epoch": 2, "train_loss": 0.2645607888698578, "cos_r_negg": [0.09845705330371857, 0.1395694762468338, 0.13869507610797882], "cos_innovation_negg": [0.38810995221138, 0.3962540924549103, 0.7656185030937195], "cos_apical_negg": [0.09845705330371857, 0.1395694613456726, 0.13869506120681763], "cos_Ac_negg": [0.38810813426971436, 0.3962060213088989, 0.7655527591705322], "r_norm": [0.7038515210151672, 0.7138141393661499, 0.7348631620407104]}, {"step": 1200, "epoch": 2, "train_loss": 0.2265443503856659, "cos_r_negg": [0.11234335601329803, 0.11361578851938248, 0.13047271966934204], "cos_innovation_negg": [0.37416985630989075, 0.4150465130805969, 0.7350255250930786], "cos_apical_negg": [0.11234335601329803, 0.11361579596996307, 0.13047271966934204], "cos_Ac_negg": [0.3741361200809479, 0.41503775119781494, 0.7350165843963623], "r_norm": [0.6068500280380249, 0.6156581044197083, 0.6326056718826294]}, {"step": 1300, "epoch": 2, "train_loss": 0.31248822808265686, "cos_r_negg": [0.10859152674674988, 0.12851887941360474, 0.22344902157783508], "cos_innovation_negg": [0.3752672076225281, 0.3706457316875458, 0.6036753058433533], "cos_apical_negg": [0.10859152674674988, 0.12851889431476593, 0.22344902157783508], "cos_Ac_negg": [0.37523192167282104, 0.37063291668891907, 0.6034502387046814], "r_norm": [0.6378560066223145, 0.646695613861084, 0.6658786535263062]}, {"step": 1400, "epoch": 2, "train_loss": 0.22328069806098938, "cos_r_negg": [0.10234048962593079, 0.11358051002025604, 0.07161962985992432], "cos_innovation_negg": [0.3311741054058075, 0.36670809984207153, 0.6587581634521484], "cos_apical_negg": [0.10234048962593079, 0.11358051002025604, 0.07161962985992432], "cos_Ac_negg": [0.3311992883682251, 0.3667413592338562, 0.6588879823684692], "r_norm": [0.7144718170166016, 0.7270647287368774, 0.7500253915786743]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9388, "test_loss": 0.2344582160949707}, {"step": 1500, "epoch": 3, "train_loss": 0.15693561732769012, "cos_r_negg": [0.10787053406238556, 0.12390090525150299, 0.2679864168167114], "cos_innovation_negg": [0.3898828625679016, 0.3926648199558258, 0.7474457621574402], "cos_apical_negg": [0.10787053406238556, 0.1239008903503418, 0.2679864168167114], "cos_Ac_negg": [0.3898441791534424, 0.3925970792770386, 0.7470107674598694], "r_norm": [0.6114495992660522, 0.619301438331604, 0.6350533962249756], "ldr": {"dL_sdil": -0.015088126063346863, "dL_bp": -0.02963070571422577, "ratio": 0.5092057613768888}}, {"step": 1600, "epoch": 3, "train_loss": 0.17743758857250214, "cos_r_negg": [0.10768801718950272, 0.1175634041428566, 0.25766721367836], "cos_innovation_negg": [0.40519052743911743, 0.4071451723575592, 0.7698088884353638], "cos_apical_negg": [0.10768801718950272, 0.1175634115934372, 0.25766721367836], "cos_Ac_negg": [0.4052160978317261, 0.4071645140647888, 0.7694416642189026], "r_norm": [0.5734442472457886, 0.5845726728439331, 0.5984963774681091]}, {"step": 1700, "epoch": 3, "train_loss": 0.20792122185230255, "cos_r_negg": [0.08450322598218918, 0.10385923087596893, 0.12117033451795578], "cos_innovation_negg": [0.37098291516304016, 0.4234551191329956, 0.7653336524963379], "cos_apical_negg": [0.08450322598218918, 0.10385923087596893, 0.12117032706737518], "cos_Ac_negg": [0.3709849715232849, 0.4234609007835388, 0.7655013203620911], "r_norm": [0.5625088214874268, 0.5712831616401672, 0.5836696624755859]}, {"step": 1800, "epoch": 3, "train_loss": 0.09999382495880127, "cos_r_negg": [0.09550761431455612, 0.12660077214241028, 0.20088191330432892], "cos_innovation_negg": [0.38705819845199585, 0.41634225845336914, 0.7071832418441772], "cos_apical_negg": [0.09550760686397552, 0.12660077214241028, 0.20088191330432892], "cos_Ac_negg": [0.3868919014930725, 0.4159446954727173, 0.7074464559555054], "r_norm": [0.557034969329834, 0.5668622255325317, 0.5829949378967285]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.94, "test_loss": 0.25001368255615236}, {"step": 1900, "epoch": 4, "train_loss": 0.15469612181186676, "cos_r_negg": [0.09818380326032639, 0.1273515820503235, 0.12134392559528351], "cos_innovation_negg": [0.41390261054039, 0.44023215770721436, 0.6958903074264526], "cos_apical_negg": [0.09818381071090698, 0.1273515820503235, 0.12134392559528351], "cos_Ac_negg": [0.41387927532196045, 0.4401131272315979, 0.6958767175674438], "r_norm": [0.5500970482826233, 0.5569380521774292, 0.574651300907135]}, {"step": 2000, "epoch": 4, "train_loss": 0.2128089964389801, "cos_r_negg": [0.09326376765966415, 0.10330966860055923, 0.24521704018115997], "cos_innovation_negg": [0.40592893958091736, 0.4548645317554474, 0.7100414037704468], "cos_apical_negg": [0.09326376765966415, 0.10330966860055923, 0.24521702527999878], "cos_Ac_negg": [0.4060746431350708, 0.4549553692340851, 0.7102867364883423], "r_norm": [0.46225154399871826, 0.46613821387290955, 0.4803006052970886], "ldr": {"dL_sdil": -0.009972944855690002, "dL_bp": -0.022860616445541382, "ratio": 0.4362500407391715}}, {"step": 2100, "epoch": 4, "train_loss": 0.15295618772506714, "cos_r_negg": [0.08869123458862305, 0.11383302509784698, 0.06687113642692566], "cos_innovation_negg": [0.39854124188423157, 0.4167359471321106, 0.7585870027542114], "cos_apical_negg": [0.08869124203920364, 0.11383302509784698, 0.06687113642692566], "cos_Ac_negg": [0.39873385429382324, 0.41682982444763184, 0.7589167356491089], "r_norm": [0.46618640422821045, 0.4707163870334625, 0.4836179316043854]}, {"step": 2200, "epoch": 4, "train_loss": 0.09179909527301788, "cos_r_negg": [0.10070512443780899, 0.12419068813323975, 0.06317576766014099], "cos_innovation_negg": [0.36842644214630127, 0.3992958068847656, 0.6719766855239868], "cos_apical_negg": [0.1007051169872284, 0.12419068813323975, 0.06317576766014099], "cos_Ac_negg": [0.3684704005718231, 0.39928698539733887, 0.672103762626648], "r_norm": [0.5058722496032715, 0.508932888507843, 0.5246779322624207]}, {"step": 2300, "epoch": 4, "train_loss": 0.2925705909729004, "cos_r_negg": [0.0751347541809082, 0.11288592964410782, 0.29184451699256897], "cos_innovation_negg": [0.40606626868247986, 0.45072969794273376, 0.8423775434494019], "cos_apical_negg": [0.0751347541809082, 0.11288593709468842, 0.29184451699256897], "cos_Ac_negg": [0.4059680104255676, 0.4505543112754822, 0.842880368232727], "r_norm": [0.4347483217716217, 0.44089943170547485, 0.45825356245040894]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9471, "test_loss": 0.21272765579223632}, {"step": 2400, "epoch": 5, "train_loss": 0.27907389402389526, "cos_r_negg": [0.08019134402275085, 0.11235351115465164, 0.11302737146615982], "cos_innovation_negg": [0.37619203329086304, 0.42445898056030273, 0.6897738575935364], "cos_apical_negg": [0.08019134402275085, 0.11235351115465164, 0.11302736401557922], "cos_Ac_negg": [0.37614938616752625, 0.4244285225868225, 0.6896795034408569], "r_norm": [0.5223556756973267, 0.5312448740005493, 0.5470421314239502]}, {"step": 2500, "epoch": 5, "train_loss": 0.16260740160942078, "cos_r_negg": [0.0909058228135109, 0.08839759975671768, 0.31978344917297363], "cos_innovation_negg": [0.440868616104126, 0.4875530004501343, 0.8524196743965149], "cos_apical_negg": [0.0909058228135109, 0.08839759230613708, 0.31978344917297363], "cos_Ac_negg": [0.44042080640792847, 0.48698002099990845, 0.851769208908081], "r_norm": [0.40238407254219055, 0.40660905838012695, 0.4197536110877991], "ldr": {"dL_sdil": -0.007677853107452393, "dL_bp": -0.019469276070594788, "ratio": 0.3943574008408333}}, {"step": 2600, "epoch": 5, "train_loss": 0.23229701817035675, "cos_r_negg": [0.06360861659049988, 0.09395895898342133, 0.18296802043914795], "cos_innovation_negg": [0.4320843815803528, 0.46038222312927246, 0.842450737953186], "cos_apical_negg": [0.06360861659049988, 0.09395897388458252, 0.18296802043914795], "cos_Ac_negg": [0.4320768117904663, 0.46017730236053467, 0.8418355584144592], "r_norm": [0.44097065925598145, 0.4436109960079193, 0.45734941959381104]}, {"step": 2700, "epoch": 5, "train_loss": 0.13178721070289612, "cos_r_negg": [0.08140005171298981, 0.11266081035137177, 0.30397453904151917], "cos_innovation_negg": [0.4453584551811218, 0.4734402596950531, 0.8197774291038513], "cos_apical_negg": [0.08140003681182861, 0.11266081035137177, 0.3039745092391968], "cos_Ac_negg": [0.44556429982185364, 0.4735179543495178, 0.8196971416473389], "r_norm": [0.48501238226890564, 0.4915286898612976, 0.5090052485466003]}, {"step": 2800, "epoch": 5, "train_loss": 0.3718406558036804, "cos_r_negg": [0.09403565526008606, 0.14480730891227722, 0.19146648049354553], "cos_innovation_negg": [0.4020455479621887, 0.459244042634964, 0.8356913924217224], "cos_apical_negg": [0.09403566271066666, 0.14480730891227722, 0.19146648049354553], "cos_Ac_negg": [0.40191662311553955, 0.4590546190738678, 0.8354898691177368], "r_norm": [0.4042191505432129, 0.4093473553657532, 0.42419520020484924]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9379, "test_loss": 0.2555447769165039}, {"step": 2900, "epoch": 6, "train_loss": 0.07557213306427002, "cos_r_negg": [0.07505618035793304, 0.08974916487932205, 0.14667253196239471], "cos_innovation_negg": [0.38145482540130615, 0.4281131327152252, 0.7992935180664062], "cos_apical_negg": [0.07505617290735245, 0.08974916487932205, 0.14667253196239471], "cos_Ac_negg": [0.38156557083129883, 0.4278411865234375, 0.8002254962921143], "r_norm": [0.3834804594516754, 0.38793498277664185, 0.40039727091789246]}, {"step": 3000, "epoch": 6, "train_loss": 0.2219008505344391, "cos_r_negg": [0.06788243353366852, 0.09483136236667633, 0.18495909869670868], "cos_innovation_negg": [0.3990977108478546, 0.45599955320358276, 0.7706533670425415], "cos_apical_negg": [0.06788243353366852, 0.09483136236667633, 0.18495911359786987], "cos_Ac_negg": [0.3992325961589813, 0.4560081958770752, 0.7704576253890991], "r_norm": [0.39800089597702026, 0.4035380482673645, 0.4163855314254761], "ldr": {"dL_sdil": -0.009042501449584961, "dL_bp": -0.019549831748008728, "ratio": 0.4625360241530465}}, {"step": 3100, "epoch": 6, "train_loss": 0.15087325870990753, "cos_r_negg": [0.05583612620830536, 0.06717230379581451, 0.21009361743927002], "cos_innovation_negg": [0.38338279724121094, 0.41219210624694824, 0.8622269630432129], "cos_apical_negg": [0.05583612620830536, 0.06717230379581451, 0.21009361743927002], "cos_Ac_negg": [0.3834088444709778, 0.41177675127983093, 0.8619050979614258], "r_norm": [0.31824159622192383, 0.3232581615447998, 0.33147305250167847]}, {"step": 3200, "epoch": 6, "train_loss": 0.21398979425430298, "cos_r_negg": [0.0641307383775711, 0.08751078695058823, 0.2056616246700287], "cos_innovation_negg": [0.38996827602386475, 0.4251745641231537, 0.7033661603927612], "cos_apical_negg": [0.0641307383775711, 0.08751077950000763, 0.2056616246700287], "cos_Ac_negg": [0.3899790048599243, 0.4249228835105896, 0.7024281024932861], "r_norm": [0.37234026193618774, 0.3795733153820038, 0.39143645763397217]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9488, "test_loss": 0.22057864151000978}, {"step": 3300, "epoch": 7, "train_loss": 0.1614992916584015, "cos_r_negg": [0.08768826723098755, 0.06972533464431763, 0.20678718388080597], "cos_innovation_negg": [0.3783720135688782, 0.41138485074043274, 0.8892285823822021], "cos_apical_negg": [0.08768826723098755, 0.06972533464431763, 0.20678719878196716], "cos_Ac_negg": [0.3784742057323456, 0.41120919585227966, 0.8892532587051392], "r_norm": [0.37459859251976013, 0.37855738401412964, 0.39130234718322754]}, {"step": 3400, "epoch": 7, "train_loss": 0.08100332319736481, "cos_r_negg": [0.08259744942188263, 0.0747326910495758, 0.17624694108963013], "cos_innovation_negg": [0.41565901041030884, 0.4770714342594147, 0.8280069231987], "cos_apical_negg": [0.08259744197130203, 0.0747326910495758, 0.17624694108963013], "cos_Ac_negg": [0.4154513478279114, 0.4769425690174103, 0.8282555341720581], "r_norm": [0.40025413036346436, 0.40759968757629395, 0.42241206765174866]}, {"step": 3500, "epoch": 7, "train_loss": 0.13410425186157227, "cos_r_negg": [0.0804743841290474, 0.0817059725522995, 0.3010490834712982], "cos_innovation_negg": [0.40426409244537354, 0.4392186999320984, 0.7740101218223572], "cos_apical_negg": [0.0804743841290474, 0.0817059725522995, 0.30104905366897583], "cos_Ac_negg": [0.4043189287185669, 0.43930932879447937, 0.7747332453727722], "r_norm": [0.4346146583557129, 0.4409008324146271, 0.4544880986213684], "ldr": {"dL_sdil": -0.014603078365325928, "dL_bp": -0.03912034630775452, "ratio": 0.37328601977205084}}, {"step": 3600, "epoch": 7, "train_loss": 0.07714975625276566, "cos_r_negg": [0.06835495680570602, 0.06928538531064987, 0.08564530313014984], "cos_innovation_negg": [0.31875497102737427, 0.40684059262275696, 0.7801216840744019], "cos_apical_negg": [0.06835495680570602, 0.06928537786006927, 0.08564530313014984], "cos_Ac_negg": [0.3182862401008606, 0.4071730971336365, 0.780643880367279], "r_norm": [0.35515639185905457, 0.3603340983390808, 0.3733851909637451]}, {"step": 3700, "epoch": 7, "train_loss": 0.07337075471878052, "cos_r_negg": [0.06587930768728256, 0.08804114162921906, 0.15163102746009827], "cos_innovation_negg": [0.374573677778244, 0.4521702527999878, 0.6877122521400452], "cos_apical_negg": [0.06587930768728256, 0.08804114162921906, 0.15163102746009827], "cos_Ac_negg": [0.3747364282608032, 0.45205315947532654, 0.6875947713851929], "r_norm": [0.3566398322582245, 0.36068716645240784, 0.371725857257843]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9515, "test_loss": 0.19599117889404297}, {"step": 3800, "epoch": 8, "train_loss": 0.13576091825962067, "cos_r_negg": [0.05338773876428604, 0.06723623722791672, 0.17333054542541504], "cos_innovation_negg": [0.34853434562683105, 0.41022980213165283, 0.8080852031707764], "cos_apical_negg": [0.05338773876428604, 0.06723624467849731, 0.17333053052425385], "cos_Ac_negg": [0.34890830516815186, 0.4100467562675476, 0.8070822954177856], "r_norm": [0.34986940026283264, 0.3551466763019562, 0.36433926224708557]}, {"step": 3900, "epoch": 8, "train_loss": 0.3243873715400696, "cos_r_negg": [0.060662999749183655, 0.08019344508647919, 0.13761413097381592], "cos_innovation_negg": [0.3742068409919739, 0.4473201632499695, 0.8598663806915283], "cos_apical_negg": [0.060662996023893356, 0.08019345253705978, 0.13761413097381592], "cos_Ac_negg": [0.3748463988304138, 0.4483766555786133, 0.8600820302963257], "r_norm": [0.34177011251449585, 0.34795674681663513, 0.3583543002605438]}, {"step": 4000, "epoch": 8, "train_loss": 0.10624215006828308, "cos_r_negg": [0.07240407913923264, 0.08963578939437866, 0.20086120069026947], "cos_innovation_negg": [0.3990883231163025, 0.48925915360450745, 0.801177978515625], "cos_apical_negg": [0.07240407168865204, 0.08963578939437866, 0.20086120069026947], "cos_Ac_negg": [0.3991803526878357, 0.4887305796146393, 0.8014171123504639], "r_norm": [0.349206805229187, 0.3547331988811493, 0.3662940263748169], "ldr": {"dL_sdil": -0.011774986982345581, "dL_bp": -0.028448879718780518, "ratio": 0.41389984768266036}}, {"step": 4100, "epoch": 8, "train_loss": 0.13333386182785034, "cos_r_negg": [0.06914068758487701, 0.08643893897533417, 0.05814060941338539], "cos_innovation_negg": [0.35781151056289673, 0.4302525520324707, 0.734013020992279], "cos_apical_negg": [0.06914068758487701, 0.08643893897533417, 0.05814060568809509], "cos_Ac_negg": [0.35766011476516724, 0.43010398745536804, 0.733878493309021], "r_norm": [0.4042797088623047, 0.40916305780410767, 0.4226120710372925]}, {"step": 4200, "epoch": 8, "train_loss": 0.2944686710834503, "cos_r_negg": [0.07071380317211151, 0.11774702370166779, 0.2421034723520279], "cos_innovation_negg": [0.38776469230651855, 0.4611736536026001, 0.8230035305023193], "cos_apical_negg": [0.07071381062269211, 0.11774702370166779, 0.2421034574508667], "cos_Ac_negg": [0.38808488845825195, 0.46097224950790405, 0.8239763379096985], "r_norm": [0.42499101161956787, 0.43017786741256714, 0.4416730999946594]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9531, "test_loss": 0.20775283966064453}, {"step": 4300, "epoch": 9, "train_loss": 0.08550158143043518, "cos_r_negg": [0.05379042774438858, 0.0747293084859848, 0.17267200350761414], "cos_innovation_negg": [0.37589749693870544, 0.4667752981185913, 0.8623870015144348], "cos_apical_negg": [0.05379042401909828, 0.0747293159365654, 0.17267200350761414], "cos_Ac_negg": [0.3758904039859772, 0.4669952988624573, 0.862406849861145], "r_norm": [0.32288748025894165, 0.3278752565383911, 0.33866462111473083]}, {"step": 4400, "epoch": 9, "train_loss": 0.04019725322723389, "cos_r_negg": [0.05765800178050995, 0.05524839460849762, 0.21880260109901428], "cos_innovation_negg": [0.40567314624786377, 0.47482752799987793, 0.8458130359649658], "cos_apical_negg": [0.05765800178050995, 0.05524839088320732, 0.21880263090133667], "cos_Ac_negg": [0.40560752153396606, 0.4751845598220825, 0.8455036878585815], "r_norm": [0.3023514449596405, 0.3072102963924408, 0.31525954604148865]}, {"step": 4500, "epoch": 9, "train_loss": 0.20656368136405945, "cos_r_negg": [0.061169013381004333, 0.0632934719324112, 0.2320057451725006], "cos_innovation_negg": [0.36124590039253235, 0.43740618228912354, 0.7263761758804321], "cos_apical_negg": [0.061169013381004333, 0.0632934719324112, 0.2320057451725006], "cos_Ac_negg": [0.36119648814201355, 0.4377795457839966, 0.7263678312301636], "r_norm": [0.3497084975242615, 0.353681743144989, 0.3688981533050537], "ldr": {"dL_sdil": -0.014830201864242554, "dL_bp": -0.03632296621799469, "ratio": 0.408287191504078}}, {"step": 4600, "epoch": 9, "train_loss": 0.10160844773054123, "cos_r_negg": [0.04702310636639595, 0.032225072383880615, 0.1727820187807083], "cos_innovation_negg": [0.34587329626083374, 0.4174235463142395, 0.7862191796302795], "cos_apical_negg": [0.04702310264110565, 0.032225072383880615, 0.17278200387954712], "cos_Ac_negg": [0.34580907225608826, 0.4188026785850525, 0.7855105400085449], "r_norm": [0.3018583655357361, 0.3093205392360687, 0.32058000564575195]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9536, "test_loss": 0.19554481658935546}, {"step": 4700, "epoch": 10, "train_loss": 0.10746815800666809, "cos_r_negg": [0.05174773931503296, 0.044344715774059296, 0.10797139257192612], "cos_innovation_negg": [0.3865963816642761, 0.4590938687324524, 0.9038069248199463], "cos_apical_negg": [0.05174774304032326, 0.044344715774059296, 0.10797140002250671], "cos_Ac_negg": [0.3881083130836487, 0.45977145433425903, 0.904222846031189], "r_norm": [0.2900826334953308, 0.29456502199172974, 0.3017042875289917]}, {"step": 4800, "epoch": 10, "train_loss": 0.10541199147701263, "cos_r_negg": [0.04803857207298279, 0.05671129748225212, 0.23833107948303223], "cos_innovation_negg": [0.38753464818000793, 0.4660438597202301, 0.8369371891021729], "cos_apical_negg": [0.04803857579827309, 0.05671130120754242, 0.23833104968070984], "cos_Ac_negg": [0.38963526487350464, 0.46886229515075684, 0.838510274887085], "r_norm": [0.2866842448711395, 0.29184892773628235, 0.2995152175426483]}, {"step": 4900, "epoch": 10, "train_loss": 0.09777286648750305, "cos_r_negg": [0.060669004917144775, 0.07559402287006378, 0.026874948292970657], "cos_innovation_negg": [0.3483748435974121, 0.39406460523605347, 0.8071061372756958], "cos_apical_negg": [0.06066899746656418, 0.07559402287006378, 0.026874952018260956], "cos_Ac_negg": [0.34847211837768555, 0.3932236135005951, 0.806911051273346], "r_norm": [0.3717004656791687, 0.375873863697052, 0.39112287759780884]}, {"step": 5000, "epoch": 10, "train_loss": 0.08608250319957733, "cos_r_negg": [0.03609630465507507, 0.04601597040891647, 0.11982588469982147], "cos_innovation_negg": [0.39659249782562256, 0.4835689067840576, 0.8205564618110657], "cos_apical_negg": [0.036096300929784775, 0.04601597040891647, 0.11982588469982147], "cos_Ac_negg": [0.39773911237716675, 0.48670482635498047, 0.8221338391304016], "r_norm": [0.2503602206707001, 0.2536306381225586, 0.26155564188957214], "ldr": {"dL_sdil": -0.005621433258056641, "dL_bp": -0.014837637543678284, "ratio": 0.3788630933670236}}, {"step": 5100, "epoch": 10, "train_loss": 0.1825137436389923, "cos_r_negg": [0.029130682349205017, 0.05148549750447273, 0.0962519496679306], "cos_innovation_negg": [0.3710719347000122, 0.429757297039032, 0.8482706546783447], "cos_apical_negg": [0.029130684211850166, 0.051485493779182434, 0.0962519496679306], "cos_Ac_negg": [0.37187981605529785, 0.4305475056171417, 0.848673939704895], "r_norm": [0.27795207500457764, 0.2815271317958832, 0.28976207971572876]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9562, "test_loss": 0.19900882568359374}, {"step": 5200, "epoch": 11, "train_loss": 0.031531766057014465, "cos_r_negg": [0.03655214607715607, 0.03297565504908562, 0.13179898262023926], "cos_innovation_negg": [0.36732256412506104, 0.4698495864868164, 0.8093478083610535], "cos_apical_negg": [0.036552149802446365, 0.03297565132379532, 0.13179898262023926], "cos_Ac_negg": [0.3689810037612915, 0.47376567125320435, 0.8130834102630615], "r_norm": [0.2927882969379425, 0.2969595491886139, 0.30312007665634155]}, {"step": 5300, "epoch": 11, "train_loss": 0.05474286526441574, "cos_r_negg": [0.039616089314222336, 0.03690396249294281, -0.018224237486720085], "cos_innovation_negg": [0.3556918799877167, 0.4213458299636841, 0.8614095449447632], "cos_apical_negg": [0.039616089314222336, 0.03690396249294281, -0.018224233761429787], "cos_Ac_negg": [0.35614973306655884, 0.4215202331542969, 0.8659122586250305], "r_norm": [0.30020302534103394, 0.30485332012176514, 0.31194496154785156]}, {"step": 5400, "epoch": 11, "train_loss": 0.21619296073913574, "cos_r_negg": [0.04012070596218109, 0.0759122371673584, 0.021753933280706406], "cos_innovation_negg": [0.37772315740585327, 0.43885338306427, 0.8346245884895325], "cos_apical_negg": [0.04012070968747139, 0.0759122371673584, 0.021753929555416107], "cos_Ac_negg": [0.3782368004322052, 0.4393499493598938, 0.8345948457717896], "r_norm": [0.2648906111717224, 0.2689496874809265, 0.27698737382888794]}, {"step": 5500, "epoch": 11, "train_loss": 0.16915827989578247, "cos_r_negg": [0.04510924592614174, 0.02041701227426529, 0.005804227665066719], "cos_innovation_negg": [0.36968448758125305, 0.4119541645050049, 0.7302386164665222], "cos_apical_negg": [0.04510924220085144, 0.02041701227426529, 0.005804236978292465], "cos_Ac_negg": [0.37084683775901794, 0.4130067229270935, 0.7298507690429688], "r_norm": [0.23753896355628967, 0.24243085086345673, 0.2478860467672348], "ldr": {"dL_sdil": -0.0063653141260147095, "dL_bp": -0.021118029952049255, "ratio": 0.3014160951787565}}, {"step": 5600, "epoch": 11, "train_loss": 0.05939307063817978, "cos_r_negg": [0.06129289045929909, 0.05863214284181595, 0.044176459312438965], "cos_innovation_negg": [0.3770793080329895, 0.4115693271160126, 0.8306359648704529], "cos_apical_negg": [0.06129289045929909, 0.058632150292396545, 0.04417645186185837], "cos_Ac_negg": [0.377330482006073, 0.41278302669525146, 0.8293212056159973], "r_norm": [0.31713148951530457, 0.3232504725456238, 0.3303645849227905]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9611, "test_loss": 0.19052626342773438}, {"step": 5700, "epoch": 12, "train_loss": 0.04905204102396965, "cos_r_negg": [0.032898612320423126, 0.020606981590390205, 0.0065833330154418945], "cos_innovation_negg": [0.36813652515411377, 0.40751972794532776, 0.8288460969924927], "cos_apical_negg": [0.032898612320423126, 0.020606987178325653, 0.0065833404660224915], "cos_Ac_negg": [0.36732012033462524, 0.4082815945148468, 0.8287973403930664], "r_norm": [0.28716182708740234, 0.29157522320747375, 0.30123478174209595]}, {"step": 5800, "epoch": 12, "train_loss": 0.05149498209357262, "cos_r_negg": [0.05042818933725357, 0.04561412334442139, 0.006646441295742989], "cos_innovation_negg": [0.35240402817726135, 0.4132154583930969, 0.8421091437339783], "cos_apical_negg": [0.05042818933725357, 0.045614127069711685, 0.006646446418017149], "cos_Ac_negg": [0.3520730435848236, 0.4140246510505676, 0.8436807990074158], "r_norm": [0.3047531843185425, 0.3098893165588379, 0.3177676796913147]}, {"step": 5900, "epoch": 12, "train_loss": 0.09314168989658356, "cos_r_negg": [0.03376233950257301, 0.0618138462305069, 0.07664758712053299], "cos_innovation_negg": [0.3483762741088867, 0.4451121687889099, 0.801885187625885], "cos_apical_negg": [0.03376233950257301, 0.0618138387799263, 0.07664759457111359], "cos_Ac_negg": [0.3490247130393982, 0.44558393955230713, 0.8022327423095703], "r_norm": [0.25475382804870605, 0.2583659589290619, 0.2643088400363922]}, {"step": 6000, "epoch": 12, "train_loss": 0.04574059695005417, "cos_r_negg": [0.0386682003736496, 0.04948435351252556, -0.029719388112425804], "cos_innovation_negg": [0.35256698727607727, 0.3934881389141083, 0.786099374294281], "cos_apical_negg": [0.0386682003736496, 0.04948434978723526, -0.02971940115094185], "cos_Ac_negg": [0.3538978695869446, 0.3947858214378357, 0.7860838770866394], "r_norm": [0.2947387099266052, 0.2999417185783386, 0.31020140647888184], "ldr": {"dL_sdil": -0.013723880052566528, "dL_bp": -0.03685186803340912, "ratio": 0.37240663187344397}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9558, "test_loss": 0.1946441520690918}, {"step": 6100, "epoch": 13, "train_loss": 0.07313955575227737, "cos_r_negg": [0.0473712794482708, 0.05741364508867264, 0.028849126771092415], "cos_innovation_negg": [0.41367626190185547, 0.4686368703842163, 0.8472159504890442], "cos_apical_negg": [0.047371283173561096, 0.05741364136338234, 0.028849134221673012], "cos_Ac_negg": [0.41373318433761597, 0.46893930435180664, 0.8472248315811157], "r_norm": [0.2699012756347656, 0.27334821224212646, 0.27994030714035034]}, {"step": 6200, "epoch": 13, "train_loss": 0.05021894350647926, "cos_r_negg": [0.054144278168678284, 0.04861467704176903, -0.053660497069358826], "cos_innovation_negg": [0.36075398325920105, 0.4268929660320282, 0.8821358680725098], "cos_apical_negg": [0.054144278168678284, 0.04861467704176903, -0.053660497069358826], "cos_Ac_negg": [0.3622370958328247, 0.43242549896240234, 0.8840374946594238], "r_norm": [0.27022266387939453, 0.27675312757492065, 0.2842249572277069]}, {"step": 6300, "epoch": 13, "train_loss": 0.057521119713783264, "cos_r_negg": [0.03135903924703598, 0.06436774134635925, -0.15125158429145813], "cos_innovation_negg": [0.3535681962966919, 0.426801860332489, 0.848403811454773], "cos_apical_negg": [0.03135904297232628, 0.06436774134635925, -0.15125158429145813], "cos_Ac_negg": [0.3534523844718933, 0.42822951078414917, 0.8487876653671265], "r_norm": [0.2335302233695984, 0.23840990662574768, 0.24550428986549377]}, {"step": 6400, "epoch": 13, "train_loss": 0.07028601318597794, "cos_r_negg": [0.008544020354747772, 0.04054494947195053, -0.053839512169361115], "cos_innovation_negg": [0.3367863893508911, 0.41978204250335693, 0.8259826302528381], "cos_apical_negg": [0.008544017560780048, 0.04054495319724083, -0.05383951961994171], "cos_Ac_negg": [0.3361411988735199, 0.42212969064712524, 0.8266055583953857], "r_norm": [0.23072147369384766, 0.23476827144622803, 0.2393934726715088]}, {"step": 6500, "epoch": 13, "train_loss": 0.026540828868746758, "cos_r_negg": [0.03228551149368286, 0.03877141699194908, -0.17610760033130646], "cos_innovation_negg": [0.3460446000099182, 0.4333292841911316, 0.8007405996322632], "cos_apical_negg": [0.03228551521897316, 0.03877141326665878, -0.17610760033130646], "cos_Ac_negg": [0.34589242935180664, 0.4331650137901306, 0.8005431890487671], "r_norm": [0.2234192192554474, 0.2283843755722046, 0.23382842540740967], "ldr": {"dL_sdil": -0.006777837872505188, "dL_bp": -0.01698954403400421, "ratio": 0.3989417172667783}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9603, "test_loss": 0.19238731880187987}, {"step": 6600, "epoch": 14, "train_loss": 0.041297897696495056, "cos_r_negg": [0.031721316277980804, 0.013572380878031254, 0.030148163437843323], "cos_innovation_negg": [0.35834527015686035, 0.43063098192214966, 0.8037606477737427], "cos_apical_negg": [0.031721316277980804, 0.013572381809353828, 0.03014816716313362], "cos_Ac_negg": [0.3590083718299866, 0.43182265758514404, 0.8040323257446289], "r_norm": [0.2462867796421051, 0.24983009696006775, 0.257363498210907]}, {"step": 6700, "epoch": 14, "train_loss": 0.046167030930519104, "cos_r_negg": [0.06146516650915146, 0.058398231863975525, -0.176686629652977], "cos_innovation_negg": [0.3227420747280121, 0.39037108421325684, 0.8751136660575867], "cos_apical_negg": [0.06146517023444176, 0.058398228138685226, -0.1766866147518158], "cos_Ac_negg": [0.3231213390827179, 0.39027535915374756, 0.875177800655365], "r_norm": [0.3230252265930176, 0.33182939887046814, 0.33672451972961426]}, {"step": 6800, "epoch": 14, "train_loss": 0.08576922118663788, "cos_r_negg": [0.037942297756671906, 0.03300860896706581, -0.1447712928056717], "cos_innovation_negg": [0.34488445520401, 0.4232572913169861, 0.8635647296905518], "cos_apical_negg": [0.037942301481962204, 0.03300861269235611, -0.14477130770683289], "cos_Ac_negg": [0.3450527787208557, 0.4242023825645447, 0.8650830388069153], "r_norm": [0.27866965532302856, 0.28555601835250854, 0.2935582995414734]}, {"step": 6900, "epoch": 14, "train_loss": 0.09305959939956665, "cos_r_negg": [0.02351430431008339, 0.029837053269147873, -0.12993724644184113], "cos_innovation_negg": [0.3242305517196655, 0.4072209596633911, 0.8378764986991882], "cos_apical_negg": [0.02351430431008339, 0.029837053269147873, -0.12993724644184113], "cos_Ac_negg": [0.32555752992630005, 0.41107887029647827, 0.8389134407043457], "r_norm": [0.218392014503479, 0.22300034761428833, 0.22705896198749542]}, {"step": 7000, "epoch": 14, "train_loss": 0.043245039880275726, "cos_r_negg": [0.06413209438323975, 0.05182088911533356, -0.049311086535453796], "cos_innovation_negg": [0.3618299961090088, 0.4285293221473694, 0.8689847588539124], "cos_apical_negg": [0.06413207948207855, 0.05182088911533356, -0.0493110790848732], "cos_Ac_negg": [0.3625856637954712, 0.43353021144866943, 0.8722168207168579], "r_norm": [0.2781904935836792, 0.2813512086868286, 0.2932329475879669], "ldr": {"dL_sdil": -0.010562092065811157, "dL_bp": -0.02707698941230774, "ratio": 0.39007630815153327}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9613, "test_loss": 0.20164922256469728}], "final": {"test_acc": 0.9613, "test_loss": 0.20164922256469728, "wall_s": 24.976321935653687, "cos_r_negg": [0.05517727881669998, 0.038112822920084, 0.010608313605189323], "cos_innovation_negg": [0.3540676236152649, 0.4195675253868103, 0.8055198192596436], "cos_apical_negg": [0.05517727881669998, 0.038112822920084, 0.010608317330479622], "cos_Ac_negg": [0.35732316970825195, 0.42626839876174927, 0.8111425638198853]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_raw_s0.json b/results/nuis_ctrl_v1_rho0p2_raw_s0.json new file mode 100644 index 0000000..75c79e4 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_raw_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_raw_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.016473568975925446, -0.026327233761548996, 0.029903199523687363], "cos_innovation_negg": [-0.016580017283558846, -0.026751887053251266, 0.033074796199798584], "cos_apical_negg": [-0.016473568975925446, -0.026327233761548996, 0.029903199523687363], "cos_Ac_negg": [-0.01658003404736519, -0.026751989498734474, 0.03307466208934784], "r_norm": [5.579668998718262, 5.17504358291626, 5.082968711853027], "ldr": {"dL_sdil": -0.0021271705627441406, "dL_bp": -0.21677780151367188, "ratio": 0.009812677072518344}}, {"step": 100, "epoch": 0, "train_loss": 0.3388369083404541, "cos_r_negg": [0.1721920520067215, 0.2719985246658325, 0.6371455192565918], "cos_innovation_negg": [0.4169691801071167, 0.49493300914764404, 0.7356089949607849], "cos_apical_negg": [0.1721920520067215, 0.2719985246658325, 0.6371455192565918], "cos_Ac_negg": [0.4166533648967743, 0.49484872817993164, 0.7353556156158447], "r_norm": [3.8753223419189453, 3.8276116847991943, 3.784452438354492]}, {"step": 200, "epoch": 0, "train_loss": 0.3989391326904297, "cos_r_negg": [0.14291749894618988, 0.2217380702495575, 0.6675006151199341], "cos_innovation_negg": [0.44804346561431885, 0.5322386026382446, 0.7960344552993774], "cos_apical_negg": [0.14291749894618988, 0.2217380702495575, 0.6675006151199341], "cos_Ac_negg": [0.4471680223941803, 0.5317703485488892, 0.7958534955978394], "r_norm": [3.729452610015869, 3.7228593826293945, 3.7083444595336914]}, {"step": 300, "epoch": 0, "train_loss": 0.4919128119945526, "cos_r_negg": [0.14903941750526428, 0.3034444749355316, 0.6772742867469788], "cos_innovation_negg": [0.4774431586265564, 0.5283323526382446, 0.8345513343811035], "cos_apical_negg": [0.14903941750526428, 0.3034444749355316, 0.6772742867469788], "cos_Ac_negg": [0.4773305356502533, 0.5281742811203003, 0.8342525362968445], "r_norm": [3.6902332305908203, 3.693300485610962, 3.678870439529419]}, {"step": 400, "epoch": 0, "train_loss": 0.24398618936538696, "cos_r_negg": [0.11366908252239227, 0.21099181473255157, 0.6483669281005859], "cos_innovation_negg": [0.45127472281455994, 0.5458073616027832, 0.8159490823745728], "cos_apical_negg": [0.11366908252239227, 0.21099181473255157, 0.6483669281005859], "cos_Ac_negg": [0.4512435793876648, 0.5458665490150452, 0.815899670124054], "r_norm": [3.672726631164551, 3.6565113067626953, 3.6241979598999023]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9177, "test_loss": 0.3047351959228516}, {"step": 500, "epoch": 1, "train_loss": 0.32535645365715027, "cos_r_negg": [0.08938203752040863, 0.24729885160923004, 0.6637201309204102], "cos_innovation_negg": [0.42945319414138794, 0.5236873626708984, 0.8064945936203003], "cos_apical_negg": [0.08938203752040863, 0.24729885160923004, 0.6637201309204102], "cos_Ac_negg": [0.428948312997818, 0.5231894254684448, 0.8059635758399963], "r_norm": [3.6340513229370117, 3.626866102218628, 3.603440046310425], "ldr": {"dL_sdil": -0.02090245485305786, "dL_bp": -0.02135103940963745, "ratio": 0.9789900365985411}}, {"step": 600, "epoch": 1, "train_loss": 0.2030535787343979, "cos_r_negg": [0.11638487130403519, 0.18938882648944855, 0.6376956701278687], "cos_innovation_negg": [0.39706772565841675, 0.4187910258769989, 0.7993124723434448], "cos_apical_negg": [0.11638487130403519, 0.18938882648944855, 0.6376956701278687], "cos_Ac_negg": [0.3970112204551697, 0.41888427734375, 0.7987611889839172], "r_norm": [3.6063642501831055, 3.613574981689453, 3.5962107181549072]}, {"step": 700, "epoch": 1, "train_loss": 0.23081550002098083, "cos_r_negg": [0.10733923316001892, 0.20635592937469482, 0.6265333890914917], "cos_innovation_negg": [0.4361274838447571, 0.48120707273483276, 0.8315417766571045], "cos_apical_negg": [0.10733923316001892, 0.20635592937469482, 0.6265333890914917], "cos_Ac_negg": [0.4358212649822235, 0.4807732105255127, 0.8313336372375488], "r_norm": [3.590412139892578, 3.5937178134918213, 3.5744543075561523]}, {"step": 800, "epoch": 1, "train_loss": 0.1793380081653595, "cos_r_negg": [0.10001850128173828, 0.20235097408294678, 0.6001237630844116], "cos_innovation_negg": [0.4340983033180237, 0.5832151174545288, 0.7938954830169678], "cos_apical_negg": [0.10001850128173828, 0.20235097408294678, 0.6001237630844116], "cos_Ac_negg": [0.43403226137161255, 0.5829353928565979, 0.793729305267334], "r_norm": [3.5731160640716553, 3.5769083499908447, 3.5541131496429443]}, {"step": 900, "epoch": 1, "train_loss": 0.3070145547389984, "cos_r_negg": [0.08809584379196167, 0.2878977954387665, 0.5420974493026733], "cos_innovation_negg": [0.4349724054336548, 0.5595442652702332, 0.8187193274497986], "cos_apical_negg": [0.08809584379196167, 0.2878977954387665, 0.5420974493026733], "cos_Ac_negg": [0.43500420451164246, 0.559375524520874, 0.8182679414749146], "r_norm": [3.564882516860962, 3.5766515731811523, 3.5528924465179443]}, {"epoch_end": 1, "step": 938, "test_acc": 0.941, "test_loss": 0.2362076156616211}, {"step": 1000, "epoch": 2, "train_loss": 0.10118691623210907, "cos_r_negg": [0.10788147151470184, 0.2473321557044983, 0.5856626033782959], "cos_innovation_negg": [0.4178251624107361, 0.5001124143600464, 0.8181846737861633], "cos_apical_negg": [0.10788147151470184, 0.2473321557044983, 0.5856626033782959], "cos_Ac_negg": [0.4178936183452606, 0.5000171661376953, 0.8181976675987244], "r_norm": [3.5756051540374756, 3.576016664505005, 3.5485241413116455], "ldr": {"dL_sdil": -0.016332685947418213, "dL_bp": -0.015833169221878052, "ratio": 1.031548751771688}}, {"step": 1100, "epoch": 2, "train_loss": 0.34849005937576294, "cos_r_negg": [0.08034752309322357, 0.2810874581336975, 0.6226122379302979], "cos_innovation_negg": [0.4257814288139343, 0.5177045464515686, 0.8171130418777466], "cos_apical_negg": [0.08034752309322357, 0.2810874581336975, 0.6226122379302979], "cos_Ac_negg": [0.4257396459579468, 0.5172585248947144, 0.8169691562652588], "r_norm": [3.540311574935913, 3.5426650047302246, 3.5222136974334717]}, {"step": 1200, "epoch": 2, "train_loss": 0.2266823947429657, "cos_r_negg": [0.06467395275831223, 0.21835751831531525, 0.6337820291519165], "cos_innovation_negg": [0.44710350036621094, 0.5692906379699707, 0.793481707572937], "cos_apical_negg": [0.06467395275831223, 0.21835751831531525, 0.6337820291519165], "cos_Ac_negg": [0.44699984788894653, 0.569176435470581, 0.7932552099227905], "r_norm": [3.534238815307617, 3.537778615951538, 3.5188679695129395]}, {"step": 1300, "epoch": 2, "train_loss": 0.24251389503479004, "cos_r_negg": [0.07839789986610413, 0.2222185879945755, 0.6220861077308655], "cos_innovation_negg": [0.39006054401397705, 0.4479979872703552, 0.7673890590667725], "cos_apical_negg": [0.07839789986610413, 0.2222185879945755, 0.6220861077308655], "cos_Ac_negg": [0.3897953927516937, 0.4478038251399994, 0.766154408454895], "r_norm": [3.557648181915283, 3.5613315105438232, 3.5341551303863525]}, {"step": 1400, "epoch": 2, "train_loss": 0.2785467803478241, "cos_r_negg": [0.051284730434417725, 0.18493859469890594, 0.5170494318008423], "cos_innovation_negg": [0.46572354435920715, 0.5562729835510254, 0.7977768182754517], "cos_apical_negg": [0.051284730434417725, 0.18493859469890594, 0.5170494318008423], "cos_Ac_negg": [0.46563971042633057, 0.5561054944992065, 0.7977519035339355], "r_norm": [3.5240864753723145, 3.526305675506592, 3.508495330810547]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9465, "test_loss": 0.21352959671020508}, {"step": 1500, "epoch": 3, "train_loss": 0.18744012713432312, "cos_r_negg": [0.055619604885578156, 0.24060164391994476, 0.5720882415771484], "cos_innovation_negg": [0.4105251431465149, 0.5300585031509399, 0.7873919010162354], "cos_apical_negg": [0.055619604885578156, 0.24060164391994476, 0.5720882415771484], "cos_Ac_negg": [0.41038763523101807, 0.5297716856002808, 0.787345290184021], "r_norm": [3.528489589691162, 3.531886100769043, 3.507558822631836], "ldr": {"dL_sdil": -0.009793221950531006, "dL_bp": -0.010277211666107178, "ratio": 0.9529065147921101}}, {"step": 1600, "epoch": 3, "train_loss": 0.17836226522922516, "cos_r_negg": [0.06417367607355118, 0.2679503560066223, 0.6074199080467224], "cos_innovation_negg": [0.4860021471977234, 0.58686363697052, 0.8405827283859253], "cos_apical_negg": [0.06417367607355118, 0.2679503560066223, 0.6074199080467224], "cos_Ac_negg": [0.48599350452423096, 0.5863144397735596, 0.8405444025993347], "r_norm": [3.5293936729431152, 3.530726909637451, 3.510951280593872]}, {"step": 1700, "epoch": 3, "train_loss": 0.18673688173294067, "cos_r_negg": [0.10869573056697845, 0.25152212381362915, 0.6026225090026855], "cos_innovation_negg": [0.4714401960372925, 0.6265134811401367, 0.7920746803283691], "cos_apical_negg": [0.10869573056697845, 0.25152212381362915, 0.6026225090026855], "cos_Ac_negg": [0.4713318347930908, 0.626278817653656, 0.791936993598938], "r_norm": [3.511098623275757, 3.5152230262756348, 3.495692729949951]}, {"step": 1800, "epoch": 3, "train_loss": 0.055265218019485474, "cos_r_negg": [0.06471377611160278, 0.22349515557289124, 0.5369325876235962], "cos_innovation_negg": [0.3954787850379944, 0.5510791540145874, 0.7634647488594055], "cos_apical_negg": [0.06471377611160278, 0.22349515557289124, 0.5369325876235962], "cos_Ac_negg": [0.3954269289970398, 0.5504909753799438, 0.7636757493019104], "r_norm": [3.547030448913574, 3.549400568008423, 3.5224273204803467]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9502, "test_loss": 0.21128672790527345}, {"step": 1900, "epoch": 4, "train_loss": 0.12737774848937988, "cos_r_negg": [0.06768354028463364, 0.26480770111083984, 0.43547239899635315], "cos_innovation_negg": [0.3673062324523926, 0.5113232135772705, 0.7954840064048767], "cos_apical_negg": [0.06768354028463364, 0.26480770111083984, 0.43547239899635315], "cos_Ac_negg": [0.3674268126487732, 0.5118414759635925, 0.79532790184021], "r_norm": [3.495820999145508, 3.5033693313598633, 3.4865591526031494]}, {"step": 2000, "epoch": 4, "train_loss": 0.1944933980703354, "cos_r_negg": [0.04760921373963356, 0.19205546379089355, 0.6536286473274231], "cos_innovation_negg": [0.40347981452941895, 0.5031334161758423, 0.8259584903717041], "cos_apical_negg": [0.04760921373963356, 0.19205546379089355, 0.6536286473274231], "cos_Ac_negg": [0.40325045585632324, 0.5030980110168457, 0.825656533241272], "r_norm": [3.5149765014648438, 3.5217597484588623, 3.5062880516052246], "ldr": {"dL_sdil": -0.00924140214920044, "dL_bp": -0.013545364141464233, "ratio": 0.6822557188338133}}, {"step": 2100, "epoch": 4, "train_loss": 0.12359166890382767, "cos_r_negg": [0.09322690218687057, 0.21033260226249695, 0.598766565322876], "cos_innovation_negg": [0.4374707043170929, 0.544856071472168, 0.8425313234329224], "cos_apical_negg": [0.09322690218687057, 0.21033260226249695, 0.598766565322876], "cos_Ac_negg": [0.4371655583381653, 0.5446057319641113, 0.842496395111084], "r_norm": [3.509075164794922, 3.521883010864258, 3.499817371368408]}, {"step": 2200, "epoch": 4, "train_loss": 0.089406318962574, "cos_r_negg": [0.05137008801102638, 0.233104407787323, 0.5743218660354614], "cos_innovation_negg": [0.3753688931465149, 0.512610137462616, 0.8219261169433594], "cos_apical_negg": [0.05137008801102638, 0.233104407787323, 0.5743218660354614], "cos_Ac_negg": [0.3752927780151367, 0.5121101140975952, 0.8224152326583862], "r_norm": [3.5120458602905273, 3.51446270942688, 3.493922472000122]}, {"step": 2300, "epoch": 4, "train_loss": 0.27675482630729675, "cos_r_negg": [0.02435573935508728, 0.19956326484680176, 0.5856771469116211], "cos_innovation_negg": [0.4101802706718445, 0.5675158500671387, 0.7972996234893799], "cos_apical_negg": [0.02435573935508728, 0.19956326484680176, 0.5856771469116211], "cos_Ac_negg": [0.4101858139038086, 0.5674529075622559, 0.7969062924385071], "r_norm": [3.4907426834106445, 3.4916887283325195, 3.4779436588287354]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9517, "test_loss": 0.2051895065307617}, {"step": 2400, "epoch": 5, "train_loss": 0.21961703896522522, "cos_r_negg": [0.07271640747785568, 0.22103393077850342, 0.5571286678314209], "cos_innovation_negg": [0.42161825299263, 0.5564560294151306, 0.7148803472518921], "cos_apical_negg": [0.07271640747785568, 0.22103393077850342, 0.5571286678314209], "cos_Ac_negg": [0.42144575715065, 0.554915726184845, 0.7144851684570312], "r_norm": [3.495201587677002, 3.502061128616333, 3.486966609954834]}, {"step": 2500, "epoch": 5, "train_loss": 0.08197547495365143, "cos_r_negg": [0.06409181654453278, 0.23346653580665588, 0.5467738509178162], "cos_innovation_negg": [0.4133896231651306, 0.5755782723426819, 0.8129197955131531], "cos_apical_negg": [0.06409181654453278, 0.23346653580665588, 0.5467738509178162], "cos_Ac_negg": [0.41320452094078064, 0.5753757953643799, 0.8126294612884521], "r_norm": [3.4914605617523193, 3.4903128147125244, 3.4726197719573975], "ldr": {"dL_sdil": -0.008657664060592651, "dL_bp": -0.010447308421134949, "ratio": 0.8286980446636534}}, {"step": 2600, "epoch": 5, "train_loss": 0.1034022867679596, "cos_r_negg": [0.05353444442152977, 0.1933932900428772, 0.6284575462341309], "cos_innovation_negg": [0.34818780422210693, 0.5147819519042969, 0.8150479197502136], "cos_apical_negg": [0.05353444442152977, 0.1933932900428772, 0.6284575462341309], "cos_Ac_negg": [0.3481002449989319, 0.5146735906600952, 0.8149440288543701], "r_norm": [3.501924514770508, 3.4994311332702637, 3.478313446044922]}, {"step": 2700, "epoch": 5, "train_loss": 0.14125940203666687, "cos_r_negg": [0.08057860285043716, 0.22372595965862274, 0.5369303822517395], "cos_innovation_negg": [0.4004718065261841, 0.5381162762641907, 0.8511413335800171], "cos_apical_negg": [0.08057860285043716, 0.22372595965862274, 0.5369303822517395], "cos_Ac_negg": [0.4004393517971039, 0.5380550026893616, 0.851192831993103], "r_norm": [3.4954171180725098, 3.4984278678894043, 3.483983039855957]}, {"step": 2800, "epoch": 5, "train_loss": 0.21846488118171692, "cos_r_negg": [0.06009357422590256, 0.24833229184150696, 0.5907409191131592], "cos_innovation_negg": [0.41767460107803345, 0.5634260773658752, 0.7554038763046265], "cos_apical_negg": [0.06009357422590256, 0.24833229184150696, 0.5907409191131592], "cos_Ac_negg": [0.4173479378223419, 0.5631324052810669, 0.7553071975708008], "r_norm": [3.4656124114990234, 3.469135046005249, 3.4514381885528564]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9521, "test_loss": 0.20345081100463866}, {"step": 2900, "epoch": 6, "train_loss": 0.10953014343976974, "cos_r_negg": [0.06787798553705215, 0.2287384271621704, 0.468828022480011], "cos_innovation_negg": [0.3793896436691284, 0.4630575478076935, 0.718155026435852], "cos_apical_negg": [0.06787798553705215, 0.2287384271621704, 0.468828022480011], "cos_Ac_negg": [0.37901097536087036, 0.46289223432540894, 0.7178455591201782], "r_norm": [3.484696388244629, 3.4866385459899902, 3.4722611904144287]}, {"step": 3000, "epoch": 6, "train_loss": 0.46337395906448364, "cos_r_negg": [0.06830329447984695, 0.1616249680519104, 0.3647420108318329], "cos_innovation_negg": [0.35110774636268616, 0.5082316994667053, 0.6819912195205688], "cos_apical_negg": [0.06830329447984695, 0.1616249680519104, 0.3647420108318329], "cos_Ac_negg": [0.35069969296455383, 0.5088880658149719, 0.6835923194885254], "r_norm": [3.469820499420166, 3.4758381843566895, 3.461033344268799], "ldr": {"dL_sdil": -0.004351168870925903, "dL_bp": -0.005708053708076477, "ratio": 0.7622859022453343}}, {"step": 3100, "epoch": 6, "train_loss": 0.1685410588979721, "cos_r_negg": [0.04595254734158516, 0.20454484224319458, 0.5188210606575012], "cos_innovation_negg": [0.3580610752105713, 0.47345858812332153, 0.728188157081604], "cos_apical_negg": [0.04595254734158516, 0.20454484224319458, 0.5188210606575012], "cos_Ac_negg": [0.3577389121055603, 0.47334158420562744, 0.7288005352020264], "r_norm": [3.477811813354492, 3.478243350982666, 3.4599335193634033]}, {"step": 3200, "epoch": 6, "train_loss": 0.17329390347003937, "cos_r_negg": [0.09172523021697998, 0.22572630643844604, 0.6188867092132568], "cos_innovation_negg": [0.40740275382995605, 0.5726735591888428, 0.8382785320281982], "cos_apical_negg": [0.09172523021697998, 0.22572630643844604, 0.6188867092132568], "cos_Ac_negg": [0.40736135840415955, 0.5720812678337097, 0.8378410935401917], "r_norm": [3.483748435974121, 3.4868130683898926, 3.4683408737182617]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9531, "test_loss": 0.19152970275878906}, {"step": 3300, "epoch": 7, "train_loss": 0.12548884749412537, "cos_r_negg": [0.03371473774313927, 0.19116787612438202, 0.5536788702011108], "cos_innovation_negg": [0.347528338432312, 0.5024522542953491, 0.8131461143493652], "cos_apical_negg": [0.03371473774313927, 0.19116787612438202, 0.5536788702011108], "cos_Ac_negg": [0.3474498987197876, 0.5019901990890503, 0.8122908473014832], "r_norm": [3.476494789123535, 3.4794178009033203, 3.4578332901000977]}, {"step": 3400, "epoch": 7, "train_loss": 0.2698051631450653, "cos_r_negg": [0.06270653009414673, 0.22756868600845337, 0.592083215713501], "cos_innovation_negg": [0.4125613570213318, 0.5473350882530212, 0.784447431564331], "cos_apical_negg": [0.06270653009414673, 0.22756868600845337, 0.592083215713501], "cos_Ac_negg": [0.4124089479446411, 0.5468129515647888, 0.784945011138916], "r_norm": [3.473717212677002, 3.4742119312286377, 3.45658278465271]}, {"step": 3500, "epoch": 7, "train_loss": 0.11865478754043579, "cos_r_negg": [0.033839091658592224, 0.1918744444847107, 0.40691661834716797], "cos_innovation_negg": [0.3008725643157959, 0.43055564165115356, 0.6733378171920776], "cos_apical_negg": [0.033839091658592224, 0.1918744444847107, 0.40691661834716797], "cos_Ac_negg": [0.3010721802711487, 0.4305219352245331, 0.6731608510017395], "r_norm": [3.475013256072998, 3.4807357788085938, 3.4610140323638916], "ldr": {"dL_sdil": -0.005170732736587524, "dL_bp": -0.007225096225738525, "ratio": 0.7156628195715122}}, {"step": 3600, "epoch": 7, "train_loss": 0.08846645057201385, "cos_r_negg": [0.07484865933656693, 0.20826342701911926, 0.42368367314338684], "cos_innovation_negg": [0.39259037375450134, 0.5593224763870239, 0.7920131087303162], "cos_apical_negg": [0.07484865933656693, 0.20826342701911926, 0.42368367314338684], "cos_Ac_negg": [0.3920590877532959, 0.5587590932846069, 0.7919071316719055], "r_norm": [3.4648241996765137, 3.469045639038086, 3.4543657302856445]}, {"step": 3700, "epoch": 7, "train_loss": 0.10054798424243927, "cos_r_negg": [0.0349927693605423, 0.18674367666244507, 0.3883827030658722], "cos_innovation_negg": [0.3381352424621582, 0.4853806793689728, 0.7522209882736206], "cos_apical_negg": [0.0349927693605423, 0.18674367666244507, 0.3883827030658722], "cos_Ac_negg": [0.3382090926170349, 0.4852631092071533, 0.7521555423736572], "r_norm": [3.4578139781951904, 3.464902400970459, 3.449568510055542]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9514, "test_loss": 0.19811919784545898}, {"step": 3800, "epoch": 8, "train_loss": 0.14756694436073303, "cos_r_negg": [0.0530828982591629, 0.16706420481204987, 0.46107226610183716], "cos_innovation_negg": [0.31439846754074097, 0.48531389236450195, 0.7160123586654663], "cos_apical_negg": [0.0530828982591629, 0.16706420481204987, 0.46107226610183716], "cos_Ac_negg": [0.3142688572406769, 0.48524773120880127, 0.7158101797103882], "r_norm": [3.449390411376953, 3.4537947177886963, 3.4377968311309814]}, {"step": 3900, "epoch": 8, "train_loss": 0.21295592188835144, "cos_r_negg": [0.04609540104866028, 0.1801377236843109, 0.5433958172798157], "cos_innovation_negg": [0.38611119985580444, 0.516525149345398, 0.8119463324546814], "cos_apical_negg": [0.04609540104866028, 0.1801377236843109, 0.5433958172798157], "cos_Ac_negg": [0.38675862550735474, 0.5163048505783081, 0.8117234706878662], "r_norm": [3.4833412170410156, 3.4833476543426514, 3.465080738067627]}, {"step": 4000, "epoch": 8, "train_loss": 0.18036073446273804, "cos_r_negg": [0.05877310782670975, 0.17528653144836426, 0.5795111656188965], "cos_innovation_negg": [0.3740137219429016, 0.5436674952507019, 0.8108839988708496], "cos_apical_negg": [0.05877310782670975, 0.17528653144836426, 0.5795111656188965], "cos_Ac_negg": [0.3739895224571228, 0.5435382723808289, 0.8106233477592468], "r_norm": [3.4689111709594727, 3.4703192710876465, 3.451960563659668], "ldr": {"dL_sdil": -0.007080137729644775, "dL_bp": -0.010071009397506714, "ratio": 0.7030216585238824}}, {"step": 4100, "epoch": 8, "train_loss": 0.0936855748295784, "cos_r_negg": [0.04878559708595276, 0.14551624655723572, 0.41049301624298096], "cos_innovation_negg": [0.31169065833091736, 0.47075045108795166, 0.7122496366500854], "cos_apical_negg": [0.04878559708595276, 0.14551624655723572, 0.41049301624298096], "cos_Ac_negg": [0.31158238649368286, 0.47061651945114136, 0.713679313659668], "r_norm": [3.465526580810547, 3.4687132835388184, 3.4495201110839844]}, {"step": 4200, "epoch": 8, "train_loss": 0.23818635940551758, "cos_r_negg": [0.016306698322296143, 0.21776893734931946, 0.4163423180580139], "cos_innovation_negg": [0.3794938623905182, 0.49791502952575684, 0.7926942110061646], "cos_apical_negg": [0.016306698322296143, 0.21776893734931946, 0.4163423180580139], "cos_Ac_negg": [0.37872251868247986, 0.4976539611816406, 0.7927689552307129], "r_norm": [3.458850145339966, 3.460315704345703, 3.4439399242401123]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.955, "test_loss": 0.18651178131103516}, {"step": 4300, "epoch": 9, "train_loss": 0.08469755947589874, "cos_r_negg": [0.022022685036063194, 0.22184401750564575, 0.5056993961334229], "cos_innovation_negg": [0.35894662141799927, 0.5069069862365723, 0.8342896699905396], "cos_apical_negg": [0.022022685036063194, 0.22184401750564575, 0.5056993961334229], "cos_Ac_negg": [0.35921746492385864, 0.5069506764411926, 0.8342188596725464], "r_norm": [3.465068817138672, 3.4675374031066895, 3.448363780975342]}, {"step": 4400, "epoch": 9, "train_loss": 0.08441644161939621, "cos_r_negg": [0.028903231024742126, 0.11586825549602509, 0.43870049715042114], "cos_innovation_negg": [0.34993427991867065, 0.44487541913986206, 0.7806928157806396], "cos_apical_negg": [0.028903231024742126, 0.11586825549602509, 0.43870049715042114], "cos_Ac_negg": [0.34975385665893555, 0.44477254152297974, 0.7801551818847656], "r_norm": [3.4610204696655273, 3.4679622650146484, 3.44842529296875]}, {"step": 4500, "epoch": 9, "train_loss": 0.23763245344161987, "cos_r_negg": [0.04483743757009506, 0.16346581280231476, 0.538713812828064], "cos_innovation_negg": [0.3578297197818756, 0.5398879051208496, 0.7825108766555786], "cos_apical_negg": [0.04483743757009506, 0.16346581280231476, 0.538713812828064], "cos_Ac_negg": [0.3577312231063843, 0.5398834943771362, 0.782530665397644], "r_norm": [3.4615845680236816, 3.4629898071289062, 3.4443883895874023], "ldr": {"dL_sdil": -0.005442783236503601, "dL_bp": -0.008815303444862366, "ratio": 0.6174243768858236}}, {"step": 4600, "epoch": 9, "train_loss": 0.13584284484386444, "cos_r_negg": [0.04618809372186661, 0.12648281455039978, 0.5143966674804688], "cos_innovation_negg": [0.35457372665405273, 0.488802433013916, 0.8005614280700684], "cos_apical_negg": [0.04618809372186661, 0.12648281455039978, 0.5143966674804688], "cos_Ac_negg": [0.3547212481498718, 0.4885520339012146, 0.8008222579956055], "r_norm": [3.471250534057617, 3.472142219543457, 3.4547955989837646]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9567, "test_loss": 0.19370537109375}, {"step": 4700, "epoch": 10, "train_loss": 0.12388288974761963, "cos_r_negg": [0.04427836090326309, 0.15790385007858276, 0.5705488324165344], "cos_innovation_negg": [0.40977948904037476, 0.5688140392303467, 0.7962357401847839], "cos_apical_negg": [0.04427836090326309, 0.15790385007858276, 0.5705488324165344], "cos_Ac_negg": [0.40972256660461426, 0.5687934160232544, 0.7956339120864868], "r_norm": [3.4553651809692383, 3.461555004119873, 3.4469351768493652]}, {"step": 4800, "epoch": 10, "train_loss": 0.0798642635345459, "cos_r_negg": [0.014005042612552643, 0.1993107795715332, 0.37496769428253174], "cos_innovation_negg": [0.30328428745269775, 0.4553671181201935, 0.7214771509170532], "cos_apical_negg": [0.014005042612552643, 0.1993107795715332, 0.37496769428253174], "cos_Ac_negg": [0.30473095178604126, 0.45609545707702637, 0.7215150594711304], "r_norm": [3.4687094688415527, 3.4758286476135254, 3.4603168964385986]}, {"step": 4900, "epoch": 10, "train_loss": 0.11851084232330322, "cos_r_negg": [0.05125885456800461, 0.18183550238609314, 0.5410358309745789], "cos_innovation_negg": [0.4377947449684143, 0.5575900077819824, 0.783425509929657], "cos_apical_negg": [0.05125885456800461, 0.18183550238609314, 0.5410358309745789], "cos_Ac_negg": [0.4377279281616211, 0.5575608015060425, 0.7831692695617676], "r_norm": [3.451565742492676, 3.453493118286133, 3.4385886192321777]}, {"step": 5000, "epoch": 10, "train_loss": 0.03343769535422325, "cos_r_negg": [0.046992167830467224, 0.14809100329875946, 0.40192416310310364], "cos_innovation_negg": [0.35171037912368774, 0.46964967250823975, 0.7679435014724731], "cos_apical_negg": [0.046992167830467224, 0.14809100329875946, 0.40192416310310364], "cos_Ac_negg": [0.3516887128353119, 0.46960732340812683, 0.7675899267196655], "r_norm": [3.464869976043701, 3.468376874923706, 3.4489669799804688], "ldr": {"dL_sdil": -0.0051101744174957275, "dL_bp": -0.0077325403690338135, "ratio": 0.6608661802737212}}, {"step": 5100, "epoch": 10, "train_loss": 0.25048214197158813, "cos_r_negg": [0.02246563509106636, 0.19471031427383423, 0.32953083515167236], "cos_innovation_negg": [0.3488203287124634, 0.5030003190040588, 0.7331562042236328], "cos_apical_negg": [0.02246563509106636, 0.19471031427383423, 0.32953083515167236], "cos_Ac_negg": [0.348846435546875, 0.5025877952575684, 0.7333074808120728], "r_norm": [3.4506001472473145, 3.4524917602539062, 3.4326670169830322]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9553, "test_loss": 0.1973849281311035}, {"step": 5200, "epoch": 11, "train_loss": 0.029526151716709137, "cos_r_negg": [0.06901518255472183, 0.1713683009147644, 0.4000300168991089], "cos_innovation_negg": [0.4098756015300751, 0.48163580894470215, 0.7498794794082642], "cos_apical_negg": [0.06901518255472183, 0.1713683009147644, 0.4000300168991089], "cos_Ac_negg": [0.41022875905036926, 0.4820902645587921, 0.7499216198921204], "r_norm": [3.441467761993408, 3.444317579269409, 3.4281182289123535]}, {"step": 5300, "epoch": 11, "train_loss": 0.14333248138427734, "cos_r_negg": [0.08245712518692017, 0.18008121848106384, 0.4830939471721649], "cos_innovation_negg": [0.3427019715309143, 0.4861484467983246, 0.8593589067459106], "cos_apical_negg": [0.08245712518692017, 0.18008121848106384, 0.4830939471721649], "cos_Ac_negg": [0.34280669689178467, 0.4862815737724304, 0.8594378232955933], "r_norm": [3.440645694732666, 3.4473037719726562, 3.4331471920013428]}, {"step": 5400, "epoch": 11, "train_loss": 0.25774702429771423, "cos_r_negg": [0.06123093515634537, 0.16761237382888794, 0.5060168504714966], "cos_innovation_negg": [0.3641701340675354, 0.5245863199234009, 0.7539292573928833], "cos_apical_negg": [0.06123093515634537, 0.16761237382888794, 0.5060168504714966], "cos_Ac_negg": [0.36482760310173035, 0.5246273279190063, 0.7533466219902039], "r_norm": [3.471930980682373, 3.475895881652832, 3.4576027393341064]}, {"step": 5500, "epoch": 11, "train_loss": 0.20137831568717957, "cos_r_negg": [0.014790154062211514, 0.2128017097711563, 0.46241673827171326], "cos_innovation_negg": [0.3637382388114929, 0.4812740087509155, 0.7801693677902222], "cos_apical_negg": [0.014790154062211514, 0.2128017097711563, 0.46241673827171326], "cos_Ac_negg": [0.3644803762435913, 0.4828115701675415, 0.7800079584121704], "r_norm": [3.447646141052246, 3.4518251419067383, 3.4343323707580566], "ldr": {"dL_sdil": -0.004327937960624695, "dL_bp": -0.005821391940116882, "ratio": 0.7434541438104575}}, {"step": 5600, "epoch": 11, "train_loss": 0.08066187798976898, "cos_r_negg": [0.03613896667957306, 0.19648058712482452, 0.49575281143188477], "cos_innovation_negg": [0.3911544680595398, 0.5481551885604858, 0.8146194219589233], "cos_apical_negg": [0.03613896667957306, 0.19648058712482452, 0.49575281143188477], "cos_Ac_negg": [0.3913358449935913, 0.5480250120162964, 0.8147847652435303], "r_norm": [3.458461284637451, 3.4564781188964844, 3.438340425491333]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9547, "test_loss": 0.20010665740966796}, {"step": 5700, "epoch": 12, "train_loss": 0.10322320461273193, "cos_r_negg": [0.027585340663790703, 0.1562771499156952, 0.42901039123535156], "cos_innovation_negg": [0.3205551505088806, 0.47852081060409546, 0.7474812865257263], "cos_apical_negg": [0.027585340663790703, 0.1562771499156952, 0.42901039123535156], "cos_Ac_negg": [0.3221622705459595, 0.4781225025653839, 0.7490335702896118], "r_norm": [3.4514660835266113, 3.4558229446411133, 3.437520742416382]}, {"step": 5800, "epoch": 12, "train_loss": 0.09904471784830093, "cos_r_negg": [-0.002126248786225915, 0.15922358632087708, 0.377025842666626], "cos_innovation_negg": [0.27672380208969116, 0.44702157378196716, 0.8337773084640503], "cos_apical_negg": [-0.002126248786225915, 0.15922358632087708, 0.377025842666626], "cos_Ac_negg": [0.2766464352607727, 0.44679200649261475, 0.8340064287185669], "r_norm": [3.4628639221191406, 3.4620978832244873, 3.4439620971679688]}, {"step": 5900, "epoch": 12, "train_loss": 0.07734407484531403, "cos_r_negg": [0.014376225881278515, 0.16064172983169556, 0.3750764727592468], "cos_innovation_negg": [0.32075750827789307, 0.4464177191257477, 0.7651004791259766], "cos_apical_negg": [0.014376225881278515, 0.16064172983169556, 0.3750764727592468], "cos_Ac_negg": [0.3215935528278351, 0.44536978006362915, 0.7645524740219116], "r_norm": [3.461233139038086, 3.4652247428894043, 3.447413444519043]}, {"step": 6000, "epoch": 12, "train_loss": 0.024701859802007675, "cos_r_negg": [0.010952739045023918, 0.13153177499771118, 0.43485701084136963], "cos_innovation_negg": [0.3408902883529663, 0.38688892126083374, 0.7951912879943848], "cos_apical_negg": [0.010952739045023918, 0.13153177499771118, 0.43485701084136963], "cos_Ac_negg": [0.3406761884689331, 0.3871164917945862, 0.795509934425354], "r_norm": [3.4502010345458984, 3.4490461349487305, 3.4353888034820557], "ldr": {"dL_sdil": -0.00697992742061615, "dL_bp": -0.013710647821426392, "ratio": 0.5090880833295294}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9567, "test_loss": 0.18746873474121092}, {"step": 6100, "epoch": 13, "train_loss": 0.05140694975852966, "cos_r_negg": [0.0009655402973294258, 0.1851300299167633, 0.39914822578430176], "cos_innovation_negg": [0.3021148145198822, 0.4084289073944092, 0.7972220182418823], "cos_apical_negg": [0.0009655402973294258, 0.1851300299167633, 0.39914822578430176], "cos_Ac_negg": [0.3022276759147644, 0.40826839208602905, 0.7963982820510864], "r_norm": [3.453662395477295, 3.456664800643921, 3.4398272037506104]}, {"step": 6200, "epoch": 13, "train_loss": 0.024884842336177826, "cos_r_negg": [0.02299400046467781, 0.14123308658599854, 0.49263402819633484], "cos_innovation_negg": [0.4263499975204468, 0.5489314794540405, 0.7903797626495361], "cos_apical_negg": [0.02299400046467781, 0.14123308658599854, 0.49263402819633484], "cos_Ac_negg": [0.42628806829452515, 0.5488638877868652, 0.7889189720153809], "r_norm": [3.4657716751098633, 3.463202476501465, 3.4446065425872803]}, {"step": 6300, "epoch": 13, "train_loss": 0.05869191884994507, "cos_r_negg": [0.01883682608604431, 0.14145004749298096, 0.5869549512863159], "cos_innovation_negg": [0.3940189480781555, 0.5031102895736694, 0.790341854095459], "cos_apical_negg": [0.01883682608604431, 0.14145004749298096, 0.5869549512863159], "cos_Ac_negg": [0.39499062299728394, 0.5030839443206787, 0.7890118956565857], "r_norm": [3.4605984687805176, 3.462883234024048, 3.4468939304351807]}, {"step": 6400, "epoch": 13, "train_loss": 0.06937310844659805, "cos_r_negg": [0.02568833716213703, 0.1709422469139099, 0.507933497428894], "cos_innovation_negg": [0.433333158493042, 0.526401162147522, 0.7905400991439819], "cos_apical_negg": [0.02568833716213703, 0.1709422469139099, 0.507933497428894], "cos_Ac_negg": [0.43355828523635864, 0.5258115530014038, 0.79041588306427], "r_norm": [3.451402187347412, 3.4500906467437744, 3.4321341514587402]}, {"step": 6500, "epoch": 13, "train_loss": 0.023130763322114944, "cos_r_negg": [0.034315600991249084, 0.18895575404167175, 0.49127936363220215], "cos_innovation_negg": [0.385489821434021, 0.5866650342941284, 0.7990735173225403], "cos_apical_negg": [0.034315600991249084, 0.18895575404167175, 0.49127936363220215], "cos_Ac_negg": [0.3861708641052246, 0.5882511138916016, 0.7986502647399902], "r_norm": [3.461559295654297, 3.4570980072021484, 3.4403011798858643], "ldr": {"dL_sdil": -0.010107576847076416, "dL_bp": -0.013823136687278748, "ratio": 0.731207183705149}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9585, "test_loss": 0.19585581665039062}, {"step": 6600, "epoch": 14, "train_loss": 0.04518500342965126, "cos_r_negg": [0.03899315744638443, 0.18686360120773315, 0.3514447510242462], "cos_innovation_negg": [0.3174074590206146, 0.4429108500480652, 0.7336504459381104], "cos_apical_negg": [0.03899315744638443, 0.18686360120773315, 0.3514447510242462], "cos_Ac_negg": [0.3177051246166229, 0.44369471073150635, 0.7339938879013062], "r_norm": [3.4517478942871094, 3.4519762992858887, 3.4337713718414307]}, {"step": 6700, "epoch": 14, "train_loss": 0.11906260251998901, "cos_r_negg": [0.002237762324512005, 0.21442167460918427, 0.47415995597839355], "cos_innovation_negg": [0.3493881821632385, 0.4919668734073639, 0.7755817174911499], "cos_apical_negg": [0.002237762324512005, 0.21442167460918427, 0.47415995597839355], "cos_Ac_negg": [0.36083731055259705, 0.49203193187713623, 0.7772910594940186], "r_norm": [3.441812515258789, 3.4441897869110107, 3.4262266159057617]}, {"step": 6800, "epoch": 14, "train_loss": 0.18277990818023682, "cos_r_negg": [0.040904827415943146, 0.15774965286254883, 0.5693460702896118], "cos_innovation_negg": [0.38608092069625854, 0.5170440077781677, 0.8491572141647339], "cos_apical_negg": [0.040904827415943146, 0.15774965286254883, 0.5693460702896118], "cos_Ac_negg": [0.3857008218765259, 0.5173830389976501, 0.8529412150382996], "r_norm": [3.4625468254089355, 3.461876392364502, 3.4417924880981445]}, {"step": 6900, "epoch": 14, "train_loss": 0.054661303758621216, "cos_r_negg": [0.014178114011883736, 0.15414831042289734, 0.4815225601196289], "cos_innovation_negg": [0.4010230004787445, 0.5195152759552002, 0.7063506841659546], "cos_apical_negg": [0.014178114011883736, 0.15414831042289734, 0.4815225601196289], "cos_Ac_negg": [0.4010884165763855, 0.5229536294937134, 0.7053012847900391], "r_norm": [3.4492599964141846, 3.450108289718628, 3.432631492614746]}, {"step": 7000, "epoch": 14, "train_loss": 0.12123630195856094, "cos_r_negg": [0.02383439615368843, 0.19232985377311707, 0.5521015524864197], "cos_innovation_negg": [0.31642913818359375, 0.5420209169387817, 0.8157789707183838], "cos_apical_negg": [0.02383439615368843, 0.19232985377311707, 0.5521015524864197], "cos_Ac_negg": [0.31504759192466736, 0.5415788292884827, 0.8173043727874756], "r_norm": [3.4482507705688477, 3.452746868133545, 3.437047004699707], "ldr": {"dL_sdil": -0.0059343427419662476, "dL_bp": -0.01396980881690979, "ratio": 0.42479770623510665}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9556, "test_loss": 0.20303017807006835}], "final": {"test_acc": 0.9556, "test_loss": 0.20303017807006835, "wall_s": 22.80018925666809, "cos_r_negg": [0.0006068069487810135, 0.14546656608581543, 0.5014218091964722], "cos_innovation_negg": [0.3084047734737396, 0.5018429756164551, 0.8085402250289917], "cos_apical_negg": [0.0006068069487810135, 0.14546656608581543, 0.5014218091964722], "cos_Ac_negg": [0.30887991189956665, 0.5017359852790833, 0.807351291179657]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_raw_s1.json b/results/nuis_ctrl_v1_rho0p2_raw_s1.json new file mode 100644 index 0000000..cf2d09f --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_raw_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_raw_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.017816580832004547, -0.012290592305362225, -0.008806847967207432], "cos_innovation_negg": [0.021385151892900467, -0.013261372223496437, -0.007389807142317295], "cos_apical_negg": [0.017816580832004547, -0.012290592305362225, -0.008806847967207432], "cos_Ac_negg": [0.021384619176387787, -0.013261749409139156, -0.007390002720057964], "r_norm": [5.40655517578125, 5.13895320892334, 5.029324531555176], "ldr": {"dL_sdil": -0.023440837860107422, "dL_bp": -0.19552373886108398, "ratio": 0.1198874264406416}}, {"step": 100, "epoch": 0, "train_loss": 0.4356670379638672, "cos_r_negg": [0.1689734160900116, 0.26602211594581604, 0.615007758140564], "cos_innovation_negg": [0.39415937662124634, 0.405183345079422, 0.8060305714607239], "cos_apical_negg": [0.1689734160900116, 0.26602211594581604, 0.615007758140564], "cos_Ac_negg": [0.39402318000793457, 0.40512290596961975, 0.8059067726135254], "r_norm": [3.8986620903015137, 3.890565872192383, 3.8833794593811035]}, {"step": 200, "epoch": 0, "train_loss": 0.38534340262413025, "cos_r_negg": [0.17627941071987152, 0.27030643820762634, 0.6998161673545837], "cos_innovation_negg": [0.4324456453323364, 0.4655967354774475, 0.8484436273574829], "cos_apical_negg": [0.17627941071987152, 0.27030643820762634, 0.6998161673545837], "cos_Ac_negg": [0.4322904348373413, 0.4656267762184143, 0.8483703136444092], "r_norm": [3.7278993129730225, 3.7348222732543945, 3.731517791748047]}, {"step": 300, "epoch": 0, "train_loss": 0.43883538246154785, "cos_r_negg": [0.1197032704949379, 0.2570406198501587, 0.5781093835830688], "cos_innovation_negg": [0.4359956383705139, 0.4337473511695862, 0.7447086572647095], "cos_apical_negg": [0.1197032704949379, 0.2570406198501587, 0.5781093835830688], "cos_Ac_negg": [0.4358632564544678, 0.433771014213562, 0.7446643114089966], "r_norm": [3.6789705753326416, 3.7146718502044678, 3.700066566467285]}, {"step": 400, "epoch": 0, "train_loss": 0.27429088950157166, "cos_r_negg": [0.12952469289302826, 0.2163768857717514, 0.5665967464447021], "cos_innovation_negg": [0.4441429376602173, 0.5018871426582336, 0.7688440680503845], "cos_apical_negg": [0.12952469289302826, 0.2163768857717514, 0.5665967464447021], "cos_Ac_negg": [0.4440825581550598, 0.5019638538360596, 0.7687709927558899], "r_norm": [3.6598286628723145, 3.6770894527435303, 3.6506810188293457]}, {"epoch_end": 0, "step": 469, "test_acc": 0.918, "test_loss": 0.32542384796142576}, {"step": 500, "epoch": 1, "train_loss": 0.39269447326660156, "cos_r_negg": [0.1062721312046051, 0.20505928993225098, 0.5644583106040955], "cos_innovation_negg": [0.42568862438201904, 0.4226480722427368, 0.7683092355728149], "cos_apical_negg": [0.1062721312046051, 0.20505928993225098, 0.5644583106040955], "cos_Ac_negg": [0.4255436062812805, 0.4224078059196472, 0.7680299282073975], "r_norm": [3.608336925506592, 3.623990535736084, 3.614894151687622], "ldr": {"dL_sdil": -0.013470113277435303, "dL_bp": -0.013411611318588257, "ratio": 1.0043620380472824}}, {"step": 600, "epoch": 1, "train_loss": 0.2612594962120056, "cos_r_negg": [0.10076740384101868, 0.20813807845115662, 0.5794553756713867], "cos_innovation_negg": [0.44401049613952637, 0.4399864077568054, 0.7417570352554321], "cos_apical_negg": [0.10076740384101868, 0.20813807845115662, 0.5794553756713867], "cos_Ac_negg": [0.4437589943408966, 0.4399217963218689, 0.7415154576301575], "r_norm": [3.6123552322387695, 3.6415977478027344, 3.628936290740967]}, {"step": 700, "epoch": 1, "train_loss": 0.2748524844646454, "cos_r_negg": [0.09547808766365051, 0.193529412150383, 0.6413819789886475], "cos_innovation_negg": [0.446560263633728, 0.577634334564209, 0.74764084815979], "cos_apical_negg": [0.09547808766365051, 0.193529412150383, 0.6413819789886475], "cos_Ac_negg": [0.4464041590690613, 0.5775347352027893, 0.7478909492492676], "r_norm": [3.6176598072052, 3.622282028198242, 3.6285762786865234]}, {"step": 800, "epoch": 1, "train_loss": 0.17241355776786804, "cos_r_negg": [0.10088097304105759, 0.27291175723075867, 0.5625531077384949], "cos_innovation_negg": [0.431984007358551, 0.5703335404396057, 0.6875219345092773], "cos_apical_negg": [0.10088097304105759, 0.27291175723075867, 0.5625531077384949], "cos_Ac_negg": [0.4318956136703491, 0.5702531337738037, 0.687503457069397], "r_norm": [3.567176103591919, 3.582148313522339, 3.5719332695007324]}, {"step": 900, "epoch": 1, "train_loss": 0.31387853622436523, "cos_r_negg": [0.09953892230987549, 0.2921331226825714, 0.5925216674804688], "cos_innovation_negg": [0.44864189624786377, 0.6091399192810059, 0.822791337966919], "cos_apical_negg": [0.09953892230987549, 0.2921331226825714, 0.5925216674804688], "cos_Ac_negg": [0.4485782980918884, 0.6092501282691956, 0.8225663304328918], "r_norm": [3.5736279487609863, 3.5842716693878174, 3.576432228088379]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9375, "test_loss": 0.25177192306518553}, {"step": 1000, "epoch": 2, "train_loss": 0.17049872875213623, "cos_r_negg": [0.10577884316444397, 0.26538562774658203, 0.5601075291633606], "cos_innovation_negg": [0.4354759454727173, 0.5381630063056946, 0.8366440534591675], "cos_apical_negg": [0.10577884316444397, 0.26538562774658203, 0.5601075291633606], "cos_Ac_negg": [0.43545007705688477, 0.5382001399993896, 0.8364940285682678], "r_norm": [3.5971617698669434, 3.6115221977233887, 3.5773520469665527], "ldr": {"dL_sdil": -0.02288752794265747, "dL_bp": -0.023692429065704346, "ratio": 0.9660270746906234}}, {"step": 1100, "epoch": 2, "train_loss": 0.3073243796825409, "cos_r_negg": [0.06935171037912369, 0.23983794450759888, 0.49295639991760254], "cos_innovation_negg": [0.4274466633796692, 0.5495652556419373, 0.724226713180542], "cos_apical_negg": [0.06935171037912369, 0.23983794450759888, 0.49295639991760254], "cos_Ac_negg": [0.42741042375564575, 0.5495082139968872, 0.7241969108581543], "r_norm": [3.55491304397583, 3.565094470977783, 3.5558106899261475]}, {"step": 1200, "epoch": 2, "train_loss": 0.3340785503387451, "cos_r_negg": [0.08844020962715149, 0.26136013865470886, 0.6662324666976929], "cos_innovation_negg": [0.4919504225254059, 0.564724326133728, 0.7682862877845764], "cos_apical_negg": [0.08844020962715149, 0.26136013865470886, 0.6662324666976929], "cos_Ac_negg": [0.49199527502059937, 0.5644968748092651, 0.7676172852516174], "r_norm": [3.5524349212646484, 3.563265562057495, 3.5444555282592773]}, {"step": 1300, "epoch": 2, "train_loss": 0.2755480408668518, "cos_r_negg": [0.0826110914349556, 0.2796480059623718, 0.7072727084159851], "cos_innovation_negg": [0.49278944730758667, 0.6161142587661743, 0.791374146938324], "cos_apical_negg": [0.0826110914349556, 0.2796480059623718, 0.7072727084159851], "cos_Ac_negg": [0.49275916814804077, 0.6161353588104248, 0.791243314743042], "r_norm": [3.5689969062805176, 3.5792593955993652, 3.56032133102417]}, {"step": 1400, "epoch": 2, "train_loss": 0.2892515957355499, "cos_r_negg": [0.04617100954055786, 0.19800525903701782, 0.5841293334960938], "cos_innovation_negg": [0.4735809564590454, 0.542998194694519, 0.8083404302597046], "cos_apical_negg": [0.04617100954055786, 0.19800525903701782, 0.5841293334960938], "cos_Ac_negg": [0.4735938310623169, 0.5430171489715576, 0.8083703517913818], "r_norm": [3.529519557952881, 3.53776478767395, 3.5263075828552246]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9436, "test_loss": 0.22161899795532228}, {"step": 1500, "epoch": 3, "train_loss": 0.18442478775978088, "cos_r_negg": [0.08789674937725067, 0.29980576038360596, 0.7146511077880859], "cos_innovation_negg": [0.4723778963088989, 0.605851411819458, 0.8120461702346802], "cos_apical_negg": [0.08789674937725067, 0.29980576038360596, 0.7146511077880859], "cos_Ac_negg": [0.47239476442337036, 0.6057537198066711, 0.8118860125541687], "r_norm": [3.5510218143463135, 3.5556280612945557, 3.5461888313293457], "ldr": {"dL_sdil": -0.00818738341331482, "dL_bp": -0.009622335433959961, "ratio": 0.8508727916945414}}, {"step": 1600, "epoch": 3, "train_loss": 0.21677272021770477, "cos_r_negg": [0.08460087329149246, 0.2268841415643692, 0.656872570514679], "cos_innovation_negg": [0.43371522426605225, 0.512417733669281, 0.8025482296943665], "cos_apical_negg": [0.08460087329149246, 0.2268841415643692, 0.656872570514679], "cos_Ac_negg": [0.43371880054473877, 0.512782633304596, 0.8022817969322205], "r_norm": [3.5290679931640625, 3.539044141769409, 3.5225727558135986]}, {"step": 1700, "epoch": 3, "train_loss": 0.11149293184280396, "cos_r_negg": [0.08747819066047668, 0.23266366124153137, 0.689258873462677], "cos_innovation_negg": [0.41761064529418945, 0.5147852301597595, 0.7864898443222046], "cos_apical_negg": [0.08747819066047668, 0.23266366124153137, 0.689258873462677], "cos_Ac_negg": [0.4176199436187744, 0.5148299932479858, 0.7862241268157959], "r_norm": [3.5327134132385254, 3.551211357116699, 3.532529830932617]}, {"step": 1800, "epoch": 3, "train_loss": 0.11369071900844574, "cos_r_negg": [0.08348599076271057, 0.24422600865364075, 0.6891183257102966], "cos_innovation_negg": [0.4450009763240814, 0.4909214675426483, 0.7027865648269653], "cos_apical_negg": [0.08348599076271057, 0.24422600865364075, 0.6891183257102966], "cos_Ac_negg": [0.4448184072971344, 0.4904190003871918, 0.7016391754150391], "r_norm": [3.567490577697754, 3.5742855072021484, 3.5611674785614014]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9467, "test_loss": 0.23838634185791016}, {"step": 1900, "epoch": 4, "train_loss": 0.11205631494522095, "cos_r_negg": [0.07415357977151871, 0.28101688623428345, 0.680112898349762], "cos_innovation_negg": [0.4501284956932068, 0.5675541162490845, 0.7829769849777222], "cos_apical_negg": [0.07415357977151871, 0.28101688623428345, 0.680112898349762], "cos_Ac_negg": [0.4500575661659241, 0.567550003528595, 0.7826475501060486], "r_norm": [3.5320348739624023, 3.548214912414551, 3.5277538299560547]}, {"step": 2000, "epoch": 4, "train_loss": 0.15781386196613312, "cos_r_negg": [0.047394976019859314, 0.24725601077079773, 0.7115209698677063], "cos_innovation_negg": [0.4403102993965149, 0.5349092483520508, 0.8005425930023193], "cos_apical_negg": [0.047394976019859314, 0.24725601077079773, 0.7115209698677063], "cos_Ac_negg": [0.44032394886016846, 0.5343272686004639, 0.800477147102356], "r_norm": [3.5300722122192383, 3.5361928939819336, 3.532763957977295], "ldr": {"dL_sdil": -0.01455448567867279, "dL_bp": -0.016517803072929382, "ratio": 0.8811393146178003}}, {"step": 2100, "epoch": 4, "train_loss": 0.1095762625336647, "cos_r_negg": [0.08439536392688751, 0.24140521883964539, 0.534023642539978], "cos_innovation_negg": [0.4613109529018402, 0.5413509607315063, 0.6667870283126831], "cos_apical_negg": [0.08439536392688751, 0.24140521883964539, 0.534023642539978], "cos_Ac_negg": [0.4611716866493225, 0.5415787696838379, 0.6719840168952942], "r_norm": [3.531738758087158, 3.5340096950531006, 3.5337023735046387]}, {"step": 2200, "epoch": 4, "train_loss": 0.15755702555179596, "cos_r_negg": [0.08772367238998413, 0.19960877299308777, 0.538733184337616], "cos_innovation_negg": [0.4514702260494232, 0.5397301912307739, 0.6183557510375977], "cos_apical_negg": [0.08772367238998413, 0.19960877299308777, 0.538733184337616], "cos_Ac_negg": [0.45148801803588867, 0.539559543132782, 0.6182036399841309], "r_norm": [3.506218433380127, 3.5101022720336914, 3.4992876052856445]}, {"step": 2300, "epoch": 4, "train_loss": 0.3984989821910858, "cos_r_negg": [0.06273116171360016, 0.19438037276268005, 0.5086185336112976], "cos_innovation_negg": [0.37528741359710693, 0.472161203622818, 0.7189158201217651], "cos_apical_negg": [0.06273116171360016, 0.19438037276268005, 0.5086185336112976], "cos_Ac_negg": [0.37531930208206177, 0.47167930006980896, 0.7185438275337219], "r_norm": [3.5103940963745117, 3.52298641204834, 3.505107879638672]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.95, "test_loss": 0.20280362167358398}, {"step": 2400, "epoch": 5, "train_loss": 0.21440386772155762, "cos_r_negg": [0.07072244584560394, 0.23109771311283112, 0.7181301116943359], "cos_innovation_negg": [0.43955373764038086, 0.5710488557815552, 0.7419193983078003], "cos_apical_negg": [0.07072244584560394, 0.23109771311283112, 0.7181301116943359], "cos_Ac_negg": [0.4395294785499573, 0.5707806944847107, 0.7417935729026794], "r_norm": [3.5005040168762207, 3.5065855979919434, 3.4987382888793945]}, {"step": 2500, "epoch": 5, "train_loss": 0.12291234731674194, "cos_r_negg": [0.08408364653587341, 0.21224187314510345, 0.671499490737915], "cos_innovation_negg": [0.46145349740982056, 0.572177529335022, 0.8321336507797241], "cos_apical_negg": [0.08408364653587341, 0.21224187314510345, 0.671499490737915], "cos_Ac_negg": [0.46149587631225586, 0.5722059011459351, 0.8317503929138184], "r_norm": [3.502736806869507, 3.506289005279541, 3.49263858795166], "ldr": {"dL_sdil": -0.01088804006576538, "dL_bp": -0.012451842427253723, "ratio": 0.8744119699053049}}, {"step": 2600, "epoch": 5, "train_loss": 0.160813570022583, "cos_r_negg": [0.06952044367790222, 0.2068316638469696, 0.6928015947341919], "cos_innovation_negg": [0.46311214566230774, 0.5328207015991211, 0.7307355403900146], "cos_apical_negg": [0.06952044367790222, 0.2068316638469696, 0.6928015947341919], "cos_Ac_negg": [0.4631936550140381, 0.5325440168380737, 0.7305200099945068], "r_norm": [3.5053138732910156, 3.5166616439819336, 3.495210647583008]}, {"step": 2700, "epoch": 5, "train_loss": 0.1364130973815918, "cos_r_negg": [0.09724610298871994, 0.2638774514198303, 0.7222675085067749], "cos_innovation_negg": [0.44005417823791504, 0.5589592456817627, 0.819959282875061], "cos_apical_negg": [0.09724610298871994, 0.2638774514198303, 0.7222675085067749], "cos_Ac_negg": [0.4400399625301361, 0.5588173866271973, 0.8197481632232666], "r_norm": [3.5365207195281982, 3.5479297637939453, 3.520289421081543]}, {"step": 2800, "epoch": 5, "train_loss": 0.25221869349479675, "cos_r_negg": [0.06721948087215424, 0.23946689069271088, 0.6013596057891846], "cos_innovation_negg": [0.4509427547454834, 0.5987998843193054, 0.7860029935836792], "cos_apical_negg": [0.06721948087215424, 0.23946689069271088, 0.6013596057891846], "cos_Ac_negg": [0.4507969617843628, 0.5983481407165527, 0.7851682305335999], "r_norm": [3.478328227996826, 3.4902567863464355, 3.4767823219299316]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9475, "test_loss": 0.20962441024780273}, {"step": 2900, "epoch": 6, "train_loss": 0.07104501128196716, "cos_r_negg": [0.050086066126823425, 0.18649601936340332, 0.5987727642059326], "cos_innovation_negg": [0.43430572748184204, 0.5571109652519226, 0.7648565173149109], "cos_apical_negg": [0.050086066126823425, 0.18649601936340332, 0.5987727642059326], "cos_Ac_negg": [0.4344123601913452, 0.5572800040245056, 0.764534056186676], "r_norm": [3.4993033409118652, 3.507411003112793, 3.4867401123046875]}, {"step": 3000, "epoch": 6, "train_loss": 0.3882247805595398, "cos_r_negg": [0.035422470420598984, 0.2375493347644806, 0.7203845977783203], "cos_innovation_negg": [0.43740540742874146, 0.5923582315444946, 0.831477701663971], "cos_apical_negg": [0.035422470420598984, 0.2375493347644806, 0.7203845977783203], "cos_Ac_negg": [0.4374907910823822, 0.592270016670227, 0.8316681385040283], "r_norm": [3.508951187133789, 3.515498399734497, 3.506570816040039], "ldr": {"dL_sdil": -0.005574941635131836, "dL_bp": -0.009410247206687927, "ratio": 0.592433069257701}}, {"step": 3100, "epoch": 6, "train_loss": 0.2562331557273865, "cos_r_negg": [0.06972925364971161, 0.21012566983699799, 0.5793290138244629], "cos_innovation_negg": [0.4135417342185974, 0.536263644695282, 0.7021746635437012], "cos_apical_negg": [0.06972925364971161, 0.21012566983699799, 0.5793290138244629], "cos_Ac_negg": [0.41355645656585693, 0.53633713722229, 0.7020848393440247], "r_norm": [3.497947931289673, 3.5051608085632324, 3.4805431365966797]}, {"step": 3200, "epoch": 6, "train_loss": 0.24760746955871582, "cos_r_negg": [0.04600472003221512, 0.2612273097038269, 0.6718626618385315], "cos_innovation_negg": [0.44852331280708313, 0.5309064984321594, 0.7615406513214111], "cos_apical_negg": [0.04600472003221512, 0.2612273097038269, 0.6718626618385315], "cos_Ac_negg": [0.44859862327575684, 0.5307490229606628, 0.7611421346664429], "r_norm": [3.4771642684936523, 3.487788677215576, 3.4756035804748535]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9516, "test_loss": 0.20080115280151367}, {"step": 3300, "epoch": 7, "train_loss": 0.0777486264705658, "cos_r_negg": [0.058215558528900146, 0.2528151571750641, 0.5984994769096375], "cos_innovation_negg": [0.4828711152076721, 0.6769227981567383, 0.798736035823822], "cos_apical_negg": [0.058215558528900146, 0.2528151571750641, 0.5984994769096375], "cos_Ac_negg": [0.4828805923461914, 0.6768467426300049, 0.7987381219863892], "r_norm": [3.482384204864502, 3.4872336387634277, 3.475127935409546]}, {"step": 3400, "epoch": 7, "train_loss": 0.2704688012599945, "cos_r_negg": [0.10562082380056381, 0.24704620242118835, 0.5503202676773071], "cos_innovation_negg": [0.42678457498550415, 0.5736896991729736, 0.6482416391372681], "cos_apical_negg": [0.10562082380056381, 0.24704620242118835, 0.5503202676773071], "cos_Ac_negg": [0.42634135484695435, 0.5731626152992249, 0.6474989652633667], "r_norm": [3.504730224609375, 3.507523536682129, 3.487412452697754]}, {"step": 3500, "epoch": 7, "train_loss": 0.1841968595981598, "cos_r_negg": [0.03964517265558243, 0.2190065085887909, 0.2805502414703369], "cos_innovation_negg": [0.3974052369594574, 0.5362122058868408, 0.7107136845588684], "cos_apical_negg": [0.03964517265558243, 0.2190065085887909, 0.2805502414703369], "cos_Ac_negg": [0.3967805504798889, 0.5360937118530273, 0.7105822563171387], "r_norm": [3.4916341304779053, 3.5076818466186523, 3.4862499237060547], "ldr": {"dL_sdil": -0.010433733463287354, "dL_bp": -0.013596653938293457, "ratio": 0.7673750843876308}}, {"step": 3600, "epoch": 7, "train_loss": 0.08116768300533295, "cos_r_negg": [0.0007512769661843777, 0.1967654526233673, 0.5335896015167236], "cos_innovation_negg": [0.4380810260772705, 0.5319277048110962, 0.7935611605644226], "cos_apical_negg": [0.0007512769661843777, 0.1967654526233673, 0.5335896015167236], "cos_Ac_negg": [0.43796706199645996, 0.5299386978149414, 0.7933510541915894], "r_norm": [3.474682331085205, 3.4827327728271484, 3.467479705810547]}, {"step": 3700, "epoch": 7, "train_loss": 0.060786016285419464, "cos_r_negg": [0.033579256385564804, 0.18584689497947693, 0.5792797803878784], "cos_innovation_negg": [0.3946492671966553, 0.5032944679260254, 0.8709812164306641], "cos_apical_negg": [0.033579256385564804, 0.18584689497947693, 0.5792797803878784], "cos_Ac_negg": [0.3946901559829712, 0.5028430223464966, 0.8706419467926025], "r_norm": [3.454129695892334, 3.4617252349853516, 3.4539918899536133]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9535, "test_loss": 0.20100207901000977}, {"step": 3800, "epoch": 8, "train_loss": 0.14766934514045715, "cos_r_negg": [0.0455578938126564, 0.199472576379776, 0.5037887692451477], "cos_innovation_negg": [0.4330657422542572, 0.5209136605262756, 0.7766220569610596], "cos_apical_negg": [0.0455578938126564, 0.199472576379776, 0.5037887692451477], "cos_Ac_negg": [0.4327586889266968, 0.5200568437576294, 0.7765603065490723], "r_norm": [3.463716983795166, 3.475459098815918, 3.4624266624450684]}, {"step": 3900, "epoch": 8, "train_loss": 0.26989418268203735, "cos_r_negg": [0.04300380498170853, 0.19460409879684448, 0.6525254249572754], "cos_innovation_negg": [0.4327391982078552, 0.5639145374298096, 0.8385655879974365], "cos_apical_negg": [0.04300380498170853, 0.19460409879684448, 0.6525254249572754], "cos_Ac_negg": [0.4327658712863922, 0.5635520219802856, 0.8384553790092468], "r_norm": [3.4783992767333984, 3.483643054962158, 3.473872423171997]}, {"step": 4000, "epoch": 8, "train_loss": 0.1700727790594101, "cos_r_negg": [0.0691879391670227, 0.27972501516342163, 0.6195012927055359], "cos_innovation_negg": [0.4578794538974762, 0.6413708925247192, 0.7179861664772034], "cos_apical_negg": [0.0691879391670227, 0.27972501516342163, 0.6195012927055359], "cos_Ac_negg": [0.4578973352909088, 0.6412854194641113, 0.7177086472511292], "r_norm": [3.4794163703918457, 3.484488010406494, 3.467482089996338], "ldr": {"dL_sdil": -0.0059860944747924805, "dL_bp": -0.008245319128036499, "ratio": 0.7259991253022586}}, {"step": 4100, "epoch": 8, "train_loss": 0.07273297011852264, "cos_r_negg": [0.06468099355697632, 0.19027966260910034, 0.5791721343994141], "cos_innovation_negg": [0.4342232942581177, 0.5594127774238586, 0.808403491973877], "cos_apical_negg": [0.06468099355697632, 0.19027966260910034, 0.5791721343994141], "cos_Ac_negg": [0.4342646896839142, 0.5592778325080872, 0.8083246350288391], "r_norm": [3.4819517135620117, 3.480591297149658, 3.471940279006958]}, {"step": 4200, "epoch": 8, "train_loss": 0.3326941728591919, "cos_r_negg": [0.0850774273276329, 0.19953115284442902, 0.6112887859344482], "cos_innovation_negg": [0.41510963439941406, 0.48489782214164734, 0.7873479127883911], "cos_apical_negg": [0.0850774273276329, 0.19953115284442902, 0.6112887859344482], "cos_Ac_negg": [0.4148026406764984, 0.48095858097076416, 0.7850573062896729], "r_norm": [3.4779555797576904, 3.482177257537842, 3.463304042816162]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9522, "test_loss": 0.19817049865722655}, {"step": 4300, "epoch": 9, "train_loss": 0.0950564593076706, "cos_r_negg": [0.07509563863277435, 0.24522316455841064, 0.6603970527648926], "cos_innovation_negg": [0.46850359439849854, 0.5816950798034668, 0.7866798639297485], "cos_apical_negg": [0.07509563863277435, 0.24522316455841064, 0.6603970527648926], "cos_Ac_negg": [0.46845144033432007, 0.5801467895507812, 0.786354660987854], "r_norm": [3.4757020473480225, 3.477034568786621, 3.471956253051758]}, {"step": 4400, "epoch": 9, "train_loss": 0.03483837842941284, "cos_r_negg": [0.0756162479519844, 0.26046431064605713, 0.5944513082504272], "cos_innovation_negg": [0.492334246635437, 0.684106707572937, 0.7453880906105042], "cos_apical_negg": [0.0756162479519844, 0.26046431064605713, 0.5944513082504272], "cos_Ac_negg": [0.4923914670944214, 0.6840108633041382, 0.7452437877655029], "r_norm": [3.470336675643921, 3.4723639488220215, 3.462059497833252]}, {"step": 4500, "epoch": 9, "train_loss": 0.1985168159008026, "cos_r_negg": [0.05870479717850685, 0.21263810992240906, 0.4580349922180176], "cos_innovation_negg": [0.4854428470134735, 0.5557766556739807, 0.626973569393158], "cos_apical_negg": [0.05870479717850685, 0.21263810992240906, 0.4580349922180176], "cos_Ac_negg": [0.4855104386806488, 0.5557011365890503, 0.6268703937530518], "r_norm": [3.484640121459961, 3.4871811866760254, 3.4654431343078613], "ldr": {"dL_sdil": -0.007090747356414795, "dL_bp": -0.011536553502082825, "ratio": 0.6146330752190957}}, {"step": 4600, "epoch": 9, "train_loss": 0.0680193081498146, "cos_r_negg": [0.04870690777897835, 0.253722608089447, 0.510011613368988], "cos_innovation_negg": [0.4142921566963196, 0.5051026344299316, 0.743492066860199], "cos_apical_negg": [0.04870690777897835, 0.253722608089447, 0.510011613368988], "cos_Ac_negg": [0.41395288705825806, 0.5022687911987305, 0.7433861494064331], "r_norm": [3.467735767364502, 3.4734983444213867, 3.462430953979492]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9538, "test_loss": 0.19944113998413085}, {"step": 4700, "epoch": 10, "train_loss": 0.09776610136032104, "cos_r_negg": [0.05886831879615784, 0.22570858895778656, 0.5670293569564819], "cos_innovation_negg": [0.4429883658885956, 0.5386934876441956, 0.6798692941665649], "cos_apical_negg": [0.05886831879615784, 0.22570858895778656, 0.5670293569564819], "cos_Ac_negg": [0.44310957193374634, 0.5391794443130493, 0.6795532703399658], "r_norm": [3.4625110626220703, 3.470669984817505, 3.455833673477173]}, {"step": 4800, "epoch": 10, "train_loss": 0.08414127677679062, "cos_r_negg": [0.050551027059555054, 0.10287302732467651, 0.3448510766029358], "cos_innovation_negg": [0.39102932810783386, 0.46998846530914307, 0.6855603456497192], "cos_apical_negg": [0.050551027059555054, 0.10287302732467651, 0.3448510766029358], "cos_Ac_negg": [0.3910481333732605, 0.46947142481803894, 0.6893148422241211], "r_norm": [3.4664206504821777, 3.472391366958618, 3.464334011077881]}, {"step": 4900, "epoch": 10, "train_loss": 0.11371088773012161, "cos_r_negg": [0.04447276517748833, 0.1574743390083313, 0.5095987319946289], "cos_innovation_negg": [0.4299008548259735, 0.5574938058853149, 0.7616347074508667], "cos_apical_negg": [0.04447276517748833, 0.1574743390083313, 0.5095987319946289], "cos_Ac_negg": [0.42991816997528076, 0.5574855804443359, 0.761502742767334], "r_norm": [3.468412160873413, 3.4738636016845703, 3.460758924484253]}, {"step": 5000, "epoch": 10, "train_loss": 0.05606111139059067, "cos_r_negg": [0.037474289536476135, 0.18544331192970276, 0.6179916262626648], "cos_innovation_negg": [0.44923120737075806, 0.6171839237213135, 0.7551383972167969], "cos_apical_negg": [0.037474289536476135, 0.18544331192970276, 0.6179916262626648], "cos_Ac_negg": [0.44958367943763733, 0.6168597340583801, 0.7548431754112244], "r_norm": [3.4625275135040283, 3.4685163497924805, 3.4564919471740723], "ldr": {"dL_sdil": -0.004449918866157532, "dL_bp": -0.0074905455112457275, "ratio": 0.5940714010050091}}, {"step": 5100, "epoch": 10, "train_loss": 0.14297005534172058, "cos_r_negg": [0.06512479484081268, 0.23240914940834045, 0.6401159167289734], "cos_innovation_negg": [0.447102427482605, 0.6113333106040955, 0.7934198379516602], "cos_apical_negg": [0.06512479484081268, 0.23240914940834045, 0.6401159167289734], "cos_Ac_negg": [0.44726502895355225, 0.6106896996498108, 0.7930235862731934], "r_norm": [3.4628477096557617, 3.4697208404541016, 3.449944496154785]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9543, "test_loss": 0.20183923797607423}, {"step": 5200, "epoch": 11, "train_loss": 0.09912631660699844, "cos_r_negg": [0.07001079618930817, 0.2584608793258667, 0.5197109580039978], "cos_innovation_negg": [0.43941575288772583, 0.5701873898506165, 0.7516946792602539], "cos_apical_negg": [0.07001079618930817, 0.2584608793258667, 0.5197109580039978], "cos_Ac_negg": [0.439582496881485, 0.5694912075996399, 0.751593828201294], "r_norm": [3.462938070297241, 3.4726412296295166, 3.448319911956787]}, {"step": 5300, "epoch": 11, "train_loss": 0.11607149988412857, "cos_r_negg": [0.05213943123817444, 0.21287769079208374, 0.4091430604457855], "cos_innovation_negg": [0.39561790227890015, 0.5195497274398804, 0.8210018277168274], "cos_apical_negg": [0.05213943123817444, 0.21287769079208374, 0.4091430604457855], "cos_Ac_negg": [0.39566001296043396, 0.5188983678817749, 0.8210616111755371], "r_norm": [3.4646692276000977, 3.4711191654205322, 3.455239772796631]}, {"step": 5400, "epoch": 11, "train_loss": 0.26368260383605957, "cos_r_negg": [0.06291709095239639, 0.24170327186584473, 0.532926619052887], "cos_innovation_negg": [0.3816204369068146, 0.5461244583129883, 0.8120216131210327], "cos_apical_negg": [0.06291709095239639, 0.24170327186584473, 0.532926619052887], "cos_Ac_negg": [0.38147851824760437, 0.5410433411598206, 0.8119370937347412], "r_norm": [3.466569185256958, 3.4699583053588867, 3.457618474960327]}, {"step": 5500, "epoch": 11, "train_loss": 0.22858896851539612, "cos_r_negg": [0.06074442341923714, 0.18535567820072174, 0.45614051818847656], "cos_innovation_negg": [0.41328179836273193, 0.5018080472946167, 0.7735607624053955], "cos_apical_negg": [0.06074442341923714, 0.18535567820072174, 0.45614051818847656], "cos_Ac_negg": [0.4130654036998749, 0.5000206828117371, 0.7733755111694336], "r_norm": [3.454829216003418, 3.459933280944824, 3.4497036933898926], "ldr": {"dL_sdil": -0.005135253071784973, "dL_bp": -0.010742738842964172, "ratio": 0.478020842436121}}, {"step": 5600, "epoch": 11, "train_loss": 0.09115181863307953, "cos_r_negg": [0.04053889214992523, 0.2067248821258545, 0.6530458927154541], "cos_innovation_negg": [0.3989574611186981, 0.5432908535003662, 0.767601728439331], "cos_apical_negg": [0.04053889214992523, 0.2067248821258545, 0.6530458927154541], "cos_Ac_negg": [0.39899998903274536, 0.5434530973434448, 0.7665030360221863], "r_norm": [3.4796388149261475, 3.4847800731658936, 3.4624600410461426]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9549, "test_loss": 0.2047551902770996}, {"step": 5700, "epoch": 12, "train_loss": 0.08562565594911575, "cos_r_negg": [0.04938215762376785, 0.22137320041656494, 0.48624658584594727], "cos_innovation_negg": [0.43018898367881775, 0.5790407657623291, 0.7335515022277832], "cos_apical_negg": [0.04938215762376785, 0.22137320041656494, 0.48624658584594727], "cos_Ac_negg": [0.43125706911087036, 0.5718858242034912, 0.7333112359046936], "r_norm": [3.468977212905884, 3.4694833755493164, 3.4570069313049316]}, {"step": 5800, "epoch": 12, "train_loss": 0.01059839129447937, "cos_r_negg": [0.05219132453203201, 0.2245350182056427, 0.6449170112609863], "cos_innovation_negg": [0.44984960556030273, 0.5593054294586182, 0.7876733541488647], "cos_apical_negg": [0.05219132453203201, 0.2245350182056427, 0.6449170112609863], "cos_Ac_negg": [0.4500811696052551, 0.556477427482605, 0.7876458168029785], "r_norm": [3.4648661613464355, 3.4633777141571045, 3.452197313308716]}, {"step": 5900, "epoch": 12, "train_loss": 0.08707671612501144, "cos_r_negg": [0.06551908701658249, 0.24000033736228943, 0.576103150844574], "cos_innovation_negg": [0.4030078947544098, 0.6000269055366516, 0.8022713661193848], "cos_apical_negg": [0.06551908701658249, 0.24000033736228943, 0.576103150844574], "cos_Ac_negg": [0.40303221344947815, 0.5962719321250916, 0.8021576404571533], "r_norm": [3.4652485847473145, 3.4664688110351562, 3.453677177429199]}, {"step": 6000, "epoch": 12, "train_loss": 0.016570743173360825, "cos_r_negg": [0.04202159494161606, 0.19206121563911438, 0.4011518955230713], "cos_innovation_negg": [0.37041717767715454, 0.5084023475646973, 0.7606583833694458], "cos_apical_negg": [0.04202159494161606, 0.19206121563911438, 0.4011518955230713], "cos_Ac_negg": [0.37013882398605347, 0.5067340135574341, 0.761455774307251], "r_norm": [3.466170310974121, 3.4691123962402344, 3.4524691104888916], "ldr": {"dL_sdil": -0.006000012159347534, "dL_bp": -0.010488852858543396, "ratio": 0.5720370225672863}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9547, "test_loss": 0.1983235137939453}, {"step": 6100, "epoch": 13, "train_loss": 0.05860435217618942, "cos_r_negg": [0.057026926428079605, 0.19623446464538574, 0.4584203362464905], "cos_innovation_negg": [0.395458847284317, 0.5979858636856079, 0.7957639694213867], "cos_apical_negg": [0.057026926428079605, 0.19623446464538574, 0.4584203362464905], "cos_Ac_negg": [0.3953757584095001, 0.5976648926734924, 0.7956451773643494], "r_norm": [3.461276054382324, 3.4666571617126465, 3.4516501426696777]}, {"step": 6200, "epoch": 13, "train_loss": 0.11902886629104614, "cos_r_negg": [0.08081215620040894, 0.1969260573387146, 0.5059528350830078], "cos_innovation_negg": [0.41033506393432617, 0.534386157989502, 0.8059005737304688], "cos_apical_negg": [0.08081215620040894, 0.1969260573387146, 0.5059528350830078], "cos_Ac_negg": [0.41017386317253113, 0.5336558818817139, 0.8060274124145508], "r_norm": [3.4563701152801514, 3.4618825912475586, 3.4465322494506836]}, {"step": 6300, "epoch": 13, "train_loss": 0.07047811150550842, "cos_r_negg": [0.029118582606315613, 0.16016680002212524, 0.527755081653595], "cos_innovation_negg": [0.39475560188293457, 0.5635925531387329, 0.8127559423446655], "cos_apical_negg": [0.029118582606315613, 0.16016680002212524, 0.527755081653595], "cos_Ac_negg": [0.39480024576187134, 0.563109815120697, 0.812713086605072], "r_norm": [3.4493629932403564, 3.452237129211426, 3.4401230812072754]}, {"step": 6400, "epoch": 13, "train_loss": 0.09686963260173798, "cos_r_negg": [0.036843959242105484, 0.19907917082309723, 0.6437855958938599], "cos_innovation_negg": [0.3797476887702942, 0.5322127938270569, 0.7541957497596741], "cos_apical_negg": [0.036843959242105484, 0.19907917082309723, 0.6437855958938599], "cos_Ac_negg": [0.3802404999732971, 0.5302999019622803, 0.7538274526596069], "r_norm": [3.4386394023895264, 3.443452835083008, 3.4289498329162598]}, {"step": 6500, "epoch": 13, "train_loss": 0.030908331274986267, "cos_r_negg": [0.025387559086084366, 0.20244021713733673, 0.5220205187797546], "cos_innovation_negg": [0.41954177618026733, 0.4961056113243103, 0.7337448000907898], "cos_apical_negg": [0.025387559086084366, 0.20244021713733673, 0.5220205187797546], "cos_Ac_negg": [0.41956377029418945, 0.495990514755249, 0.733537495136261], "r_norm": [3.462789297103882, 3.4721198081970215, 3.445420742034912], "ldr": {"dL_sdil": -0.01279139518737793, "dL_bp": -0.019137457013130188, "ratio": 0.6683957632720883}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9547, "test_loss": 0.2109088150024414}, {"step": 6600, "epoch": 14, "train_loss": 0.05749582499265671, "cos_r_negg": [0.031158369034528732, 0.14849227666854858, 0.46749401092529297], "cos_innovation_negg": [0.3765043020248413, 0.5662267208099365, 0.732268750667572], "cos_apical_negg": [0.031158369034528732, 0.14849227666854858, 0.46749401092529297], "cos_Ac_negg": [0.3768375515937805, 0.565014123916626, 0.7313430309295654], "r_norm": [3.443437337875366, 3.4483394622802734, 3.433666944503784]}, {"step": 6700, "epoch": 14, "train_loss": 0.07075092196464539, "cos_r_negg": [0.035666707903146744, 0.13649296760559082, 0.527536153793335], "cos_innovation_negg": [0.37695565819740295, 0.5032856464385986, 0.7208537459373474], "cos_apical_negg": [0.035666707903146744, 0.13649296760559082, 0.527536153793335], "cos_Ac_negg": [0.37247520685195923, 0.4940648078918457, 0.7214541435241699], "r_norm": [3.4513092041015625, 3.458841562271118, 3.4443211555480957]}, {"step": 6800, "epoch": 14, "train_loss": 0.16527383029460907, "cos_r_negg": [0.0403471365571022, 0.1276024580001831, 0.5496867895126343], "cos_innovation_negg": [0.37198305130004883, 0.4789280891418457, 0.7920528054237366], "cos_apical_negg": [0.0403471365571022, 0.1276024580001831, 0.5496867895126343], "cos_Ac_negg": [0.37165147066116333, 0.47502797842025757, 0.7918992042541504], "r_norm": [3.4475739002227783, 3.4508376121520996, 3.435202121734619]}, {"step": 6900, "epoch": 14, "train_loss": 0.05497385933995247, "cos_r_negg": [0.06007204204797745, 0.19484585523605347, 0.5344549417495728], "cos_innovation_negg": [0.42855221033096313, 0.552445650100708, 0.8151719570159912], "cos_apical_negg": [0.06007204204797745, 0.19484585523605347, 0.5344549417495728], "cos_Ac_negg": [0.42856401205062866, 0.5520523190498352, 0.8149268627166748], "r_norm": [3.450343370437622, 3.454378604888916, 3.4388420581817627]}, {"step": 7000, "epoch": 14, "train_loss": 0.1180526539683342, "cos_r_negg": [0.021235257387161255, 0.14549796283245087, 0.41770482063293457], "cos_innovation_negg": [0.40436187386512756, 0.49030813574790955, 0.746916651725769], "cos_apical_negg": [0.021235257387161255, 0.14549796283245087, 0.41770482063293457], "cos_Ac_negg": [0.4050508439540863, 0.4935333728790283, 0.7477433085441589], "r_norm": [3.452204942703247, 3.4590353965759277, 3.4412460327148438], "ldr": {"dL_sdil": -0.0052800774574279785, "dL_bp": -0.01100870966911316, "ratio": 0.4796272784123057}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9539, "test_loss": 0.2062372871398926}], "final": {"test_acc": 0.9539, "test_loss": 0.2062372871398926, "wall_s": 22.702091693878174, "cos_r_negg": [9.238254278898239e-06, 0.1693142205476761, 0.5549061298370361], "cos_innovation_negg": [0.3668328523635864, 0.518203616142273, 0.7497802972793579], "cos_apical_negg": [9.238254278898239e-06, 0.1693142205476761, 0.5549061298370361], "cos_Ac_negg": [0.36657118797302246, 0.5130937099456787, 0.7536516189575195]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_raw_s2.json b/results/nuis_ctrl_v1_rho0p2_raw_s2.json new file mode 100644 index 0000000..bfe2a2c --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_raw_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_raw_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.01100200042128563, 0.03520447760820389, 0.0008281543850898743], "cos_innovation_negg": [0.012890354730188847, 0.03915034234523773, 0.0022402864415198565], "cos_apical_negg": [0.01100200042128563, 0.03520447760820389, 0.0008281543850898743], "cos_Ac_negg": [0.012891530059278011, 0.039150483906269073, 0.002240527654066682], "r_norm": [5.454525947570801, 5.131897449493408, 5.140923500061035], "ldr": {"dL_sdil": -0.032923221588134766, "dL_bp": -0.22342252731323242, "ratio": 0.14735855862008618}}, {"step": 100, "epoch": 0, "train_loss": 0.4567059576511383, "cos_r_negg": [0.183788001537323, 0.2661236524581909, 0.5584408044815063], "cos_innovation_negg": [0.3884589672088623, 0.41925448179244995, 0.637851357460022], "cos_apical_negg": [0.183788001537323, 0.2661236524581909, 0.5584408044815063], "cos_Ac_negg": [0.38843798637390137, 0.4193718433380127, 0.6338832974433899], "r_norm": [3.8852334022521973, 3.9061994552612305, 3.876513957977295]}, {"step": 200, "epoch": 0, "train_loss": 0.5221072435379028, "cos_r_negg": [0.15865111351013184, 0.2638058066368103, 0.5799490213394165], "cos_innovation_negg": [0.45361852645874023, 0.4495181143283844, 0.7601876258850098], "cos_apical_negg": [0.15865111351013184, 0.2638058066368103, 0.5799490213394165], "cos_Ac_negg": [0.45342111587524414, 0.4498065412044525, 0.7596868276596069], "r_norm": [3.6946067810058594, 3.716439723968506, 3.7157468795776367]}, {"step": 300, "epoch": 0, "train_loss": 0.3912925720214844, "cos_r_negg": [0.12103879451751709, 0.20954403281211853, 0.3600914776325226], "cos_innovation_negg": [0.39541223645210266, 0.3972243368625641, 0.63163161277771], "cos_apical_negg": [0.12103879451751709, 0.20954403281211853, 0.3600914776325226], "cos_Ac_negg": [0.3952164649963379, 0.39704662561416626, 0.6314436197280884], "r_norm": [3.6589107513427734, 3.681612968444824, 3.675020933151245]}, {"step": 400, "epoch": 0, "train_loss": 0.2649149000644684, "cos_r_negg": [0.1183207631111145, 0.20703822374343872, 0.525892972946167], "cos_innovation_negg": [0.42131781578063965, 0.44716107845306396, 0.5986328125], "cos_apical_negg": [0.1183207631111145, 0.20703822374343872, 0.525892972946167], "cos_Ac_negg": [0.42124143242836, 0.4470200836658478, 0.5985804796218872], "r_norm": [3.680372714996338, 3.7120895385742188, 3.701115846633911]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9238, "test_loss": 0.30036927795410157}, {"step": 500, "epoch": 1, "train_loss": 0.2979006767272949, "cos_r_negg": [0.10134469717741013, 0.24083799123764038, 0.561469316482544], "cos_innovation_negg": [0.4165549874305725, 0.4379448890686035, 0.725527286529541], "cos_apical_negg": [0.10134469717741013, 0.24083799123764038, 0.561469316482544], "cos_Ac_negg": [0.4163804352283478, 0.4378066956996918, 0.7251526117324829], "r_norm": [3.6300320625305176, 3.654676675796509, 3.649027109146118], "ldr": {"dL_sdil": -0.01135137677192688, "dL_bp": -0.012096881866455078, "ratio": 0.9383721273995822}}, {"step": 600, "epoch": 1, "train_loss": 0.20395031571388245, "cos_r_negg": [0.10366348922252655, 0.2262384593486786, 0.3926882743835449], "cos_innovation_negg": [0.4073781669139862, 0.4379689395427704, 0.649882435798645], "cos_apical_negg": [0.10366348922252655, 0.2262384593486786, 0.3926882743835449], "cos_Ac_negg": [0.4072294235229492, 0.43774914741516113, 0.648726224899292], "r_norm": [3.621697425842285, 3.637753963470459, 3.62241530418396]}, {"step": 700, "epoch": 1, "train_loss": 0.27171626687049866, "cos_r_negg": [0.08675828576087952, 0.2611267864704132, 0.5247832536697388], "cos_innovation_negg": [0.429625540971756, 0.45289304852485657, 0.6150094270706177], "cos_apical_negg": [0.08675828576087952, 0.2611267864704132, 0.5247832536697388], "cos_Ac_negg": [0.42956364154815674, 0.45258814096450806, 0.6146399974822998], "r_norm": [3.6252498626708984, 3.646993637084961, 3.630019187927246]}, {"step": 800, "epoch": 1, "train_loss": 0.19473892450332642, "cos_r_negg": [0.09996411204338074, 0.2637495696544647, 0.5815935730934143], "cos_innovation_negg": [0.43061530590057373, 0.545244574546814, 0.7611676454544067], "cos_apical_negg": [0.09996411204338074, 0.2637495696544647, 0.5815935730934143], "cos_Ac_negg": [0.43070554733276367, 0.544631838798523, 0.7609244585037231], "r_norm": [3.590703010559082, 3.60972261428833, 3.605208158493042]}, {"step": 900, "epoch": 1, "train_loss": 0.3473258316516876, "cos_r_negg": [0.11643949896097183, 0.20353730022907257, 0.5414054989814758], "cos_innovation_negg": [0.42487579584121704, 0.4055999517440796, 0.6959956288337708], "cos_apical_negg": [0.11643949896097183, 0.20353730022907257, 0.5414054989814758], "cos_Ac_negg": [0.4248531460762024, 0.4055476486682892, 0.6957740783691406], "r_norm": [3.6190974712371826, 3.6391844749450684, 3.618412733078003]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9365, "test_loss": 0.2589676322937012}, {"step": 1000, "epoch": 2, "train_loss": 0.12500594556331635, "cos_r_negg": [0.08203049004077911, 0.1839422732591629, 0.48551541566848755], "cos_innovation_negg": [0.3441847860813141, 0.36872947216033936, 0.7727000713348389], "cos_apical_negg": [0.08203049004077911, 0.1839422732591629, 0.48551541566848755], "cos_Ac_negg": [0.3442431092262268, 0.36869943141937256, 0.7725566625595093], "r_norm": [3.5884664058685303, 3.610746383666992, 3.5924630165100098], "ldr": {"dL_sdil": -0.010504692792892456, "dL_bp": -0.011980056762695312, "ratio": 0.8768483322719313}}, {"step": 1100, "epoch": 2, "train_loss": 0.34199368953704834, "cos_r_negg": [0.11164788901805878, 0.23326581716537476, 0.6226155757904053], "cos_innovation_negg": [0.4554581046104431, 0.460382342338562, 0.7208415269851685], "cos_apical_negg": [0.11164788901805878, 0.23326581716537476, 0.6226155757904053], "cos_Ac_negg": [0.4554137587547302, 0.4602082669734955, 0.7207430601119995], "r_norm": [3.594935655593872, 3.6190807819366455, 3.607933521270752]}, {"step": 1200, "epoch": 2, "train_loss": 0.2814132571220398, "cos_r_negg": [0.11584852635860443, 0.2605387568473816, 0.6017051935195923], "cos_innovation_negg": [0.4351564645767212, 0.42843958735466003, 0.749901533126831], "cos_apical_negg": [0.11584852635860443, 0.2605387568473816, 0.6017051935195923], "cos_Ac_negg": [0.4350380301475525, 0.4283710718154907, 0.749815046787262], "r_norm": [3.6309621334075928, 3.6449289321899414, 3.628077507019043]}, {"step": 1300, "epoch": 2, "train_loss": 0.338234007358551, "cos_r_negg": [0.09786969423294067, 0.24103958904743195, 0.4937022924423218], "cos_innovation_negg": [0.44733133912086487, 0.5467552542686462, 0.8056282997131348], "cos_apical_negg": [0.09786969423294067, 0.24103958904743195, 0.4937022924423218], "cos_Ac_negg": [0.4471920132637024, 0.5468170642852783, 0.8056417107582092], "r_norm": [3.600451946258545, 3.6231000423431396, 3.6078546047210693]}, {"step": 1400, "epoch": 2, "train_loss": 0.279987633228302, "cos_r_negg": [0.10552861541509628, 0.196035236120224, 0.35683348774909973], "cos_innovation_negg": [0.4144078493118286, 0.42837873101234436, 0.7016053199768066], "cos_apical_negg": [0.10552861541509628, 0.196035236120224, 0.35683348774909973], "cos_Ac_negg": [0.4144124984741211, 0.4282885193824768, 0.7014795541763306], "r_norm": [3.60420560836792, 3.620573043823242, 3.616130828857422]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9439, "test_loss": 0.23199323806762695}, {"step": 1500, "epoch": 3, "train_loss": 0.21747246384620667, "cos_r_negg": [0.09274222701787949, 0.22619463503360748, 0.5033063888549805], "cos_innovation_negg": [0.4356076121330261, 0.47053012251853943, 0.6841946840286255], "cos_apical_negg": [0.09274222701787949, 0.22619463503360748, 0.5033063888549805], "cos_Ac_negg": [0.4354821443557739, 0.4703521728515625, 0.6840982437133789], "r_norm": [3.5539584159851074, 3.5746707916259766, 3.5701417922973633], "ldr": {"dL_sdil": -0.009036242961883545, "dL_bp": -0.010900884866714478, "ratio": 0.8289458215887996}}, {"step": 1600, "epoch": 3, "train_loss": 0.16448652744293213, "cos_r_negg": [0.0730118453502655, 0.10163488984107971, 0.33589762449264526], "cos_innovation_negg": [0.4109404683113098, 0.4354201555252075, 0.6562427282333374], "cos_apical_negg": [0.0730118453502655, 0.10163488984107971, 0.33589762449264526], "cos_Ac_negg": [0.41103115677833557, 0.4354763925075531, 0.6558845043182373], "r_norm": [3.548964262008667, 3.5620219707489014, 3.55721378326416]}, {"step": 1700, "epoch": 3, "train_loss": 0.11381898820400238, "cos_r_negg": [0.09348250180482864, 0.10862497240304947, 0.5267173647880554], "cos_innovation_negg": [0.45632195472717285, 0.445330411195755, 0.7073129415512085], "cos_apical_negg": [0.09348250180482864, 0.10862497240304947, 0.5267173647880554], "cos_Ac_negg": [0.45629990100860596, 0.4455423653125763, 0.7071501016616821], "r_norm": [3.564303398132324, 3.5836260318756104, 3.584378242492676]}, {"step": 1800, "epoch": 3, "train_loss": 0.12649154663085938, "cos_r_negg": [0.10802160203456879, 0.11879727989435196, 0.3618788719177246], "cos_innovation_negg": [0.385187566280365, 0.388162761926651, 0.5935660004615784], "cos_apical_negg": [0.10802160203456879, 0.11879727989435196, 0.3618788719177246], "cos_Ac_negg": [0.38517194986343384, 0.3880550265312195, 0.5938934087753296], "r_norm": [3.596954345703125, 3.6132450103759766, 3.607607364654541]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.945, "test_loss": 0.2334100471496582}, {"step": 1900, "epoch": 4, "train_loss": 0.13103771209716797, "cos_r_negg": [0.10467971116304398, 0.20130600035190582, 0.6237015724182129], "cos_innovation_negg": [0.38329052925109863, 0.4045200049877167, 0.7487303018569946], "cos_apical_negg": [0.10467971116304398, 0.20130600035190582, 0.6237015724182129], "cos_Ac_negg": [0.3832147717475891, 0.40447109937667847, 0.7478185296058655], "r_norm": [3.5451278686523438, 3.560122489929199, 3.5429205894470215]}, {"step": 2000, "epoch": 4, "train_loss": 0.25334614515304565, "cos_r_negg": [0.09000155329704285, 0.23374587297439575, 0.44203490018844604], "cos_innovation_negg": [0.42575353384017944, 0.487026184797287, 0.7367817163467407], "cos_apical_negg": [0.09000155329704285, 0.23374587297439575, 0.44203490018844604], "cos_Ac_negg": [0.42574718594551086, 0.486884206533432, 0.7365819811820984], "r_norm": [3.539418935775757, 3.553729772567749, 3.5494563579559326], "ldr": {"dL_sdil": -0.016608983278274536, "dL_bp": -0.020770937204360962, "ratio": 0.799626088840488}}, {"step": 2100, "epoch": 4, "train_loss": 0.13614681363105774, "cos_r_negg": [0.11624449491500854, 0.24827834963798523, 0.540483295917511], "cos_innovation_negg": [0.4476695656776428, 0.5435805320739746, 0.7354997396469116], "cos_apical_negg": [0.11624449491500854, 0.24827834963798523, 0.540483295917511], "cos_Ac_negg": [0.4463356137275696, 0.5433157682418823, 0.7353386878967285], "r_norm": [3.5509908199310303, 3.564251661300659, 3.5484390258789062]}, {"step": 2200, "epoch": 4, "train_loss": 0.09910304844379425, "cos_r_negg": [0.0648544654250145, 0.2561730742454529, 0.6050653457641602], "cos_innovation_negg": [0.4268740117549896, 0.5653669238090515, 0.7930992841720581], "cos_apical_negg": [0.0648544654250145, 0.2561730742454529, 0.6050653457641602], "cos_Ac_negg": [0.4268888533115387, 0.5653213262557983, 0.7929272055625916], "r_norm": [3.5276098251342773, 3.5417864322662354, 3.5346827507019043]}, {"step": 2300, "epoch": 4, "train_loss": 0.3346070349216461, "cos_r_negg": [0.05427011847496033, 0.21244634687900543, 0.36979740858078003], "cos_innovation_negg": [0.3929559886455536, 0.5010935068130493, 0.6559827327728271], "cos_apical_negg": [0.05427011847496033, 0.21244634687900543, 0.36979740858078003], "cos_Ac_negg": [0.3928688168525696, 0.5009849667549133, 0.6558835506439209], "r_norm": [3.5298454761505127, 3.5479307174682617, 3.538519859313965]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9479, "test_loss": 0.2237758575439453}, {"step": 2400, "epoch": 5, "train_loss": 0.20185178518295288, "cos_r_negg": [0.06709720939397812, 0.20028173923492432, 0.38817980885505676], "cos_innovation_negg": [0.3768389821052551, 0.46378570795059204, 0.6710083484649658], "cos_apical_negg": [0.06709720939397812, 0.20028173923492432, 0.38817980885505676], "cos_Ac_negg": [0.3767949938774109, 0.46380507946014404, 0.6708019971847534], "r_norm": [3.5429935455322266, 3.5546131134033203, 3.5520107746124268]}, {"step": 2500, "epoch": 5, "train_loss": 0.1784784197807312, "cos_r_negg": [0.0768587589263916, 0.1248767226934433, 0.34923142194747925], "cos_innovation_negg": [0.42881113290786743, 0.44066083431243896, 0.6540669202804565], "cos_apical_negg": [0.0768587589263916, 0.1248767226934433, 0.34923142194747925], "cos_Ac_negg": [0.42882055044174194, 0.4387943148612976, 0.6531764268875122], "r_norm": [3.551330089569092, 3.5617291927337646, 3.557990074157715], "ldr": {"dL_sdil": -0.0162622332572937, "dL_bp": -0.021854013204574585, "ratio": 0.7441302933728261}}, {"step": 2600, "epoch": 5, "train_loss": 0.16214726865291595, "cos_r_negg": [0.052363526076078415, 0.08490971475839615, 0.33629992604255676], "cos_innovation_negg": [0.32632553577423096, 0.3073757588863373, 0.7332020998001099], "cos_apical_negg": [0.052363526076078415, 0.08490971475839615, 0.33629992604255676], "cos_Ac_negg": [0.32628488540649414, 0.3073587417602539, 0.7331984043121338], "r_norm": [3.543248176574707, 3.558189630508423, 3.542583703994751]}, {"step": 2700, "epoch": 5, "train_loss": 0.1369170844554901, "cos_r_negg": [0.055529460310935974, 0.2342858910560608, 0.3659183979034424], "cos_innovation_negg": [0.42261722683906555, 0.4846903681755066, 0.6340969800949097], "cos_apical_negg": [0.055529460310935974, 0.2342858910560608, 0.3659183979034424], "cos_Ac_negg": [0.4226449429988861, 0.4847880005836487, 0.6337226629257202], "r_norm": [3.536177635192871, 3.552563190460205, 3.5478663444519043]}, {"step": 2800, "epoch": 5, "train_loss": 0.2521779537200928, "cos_r_negg": [0.06476972252130508, 0.24740000069141388, 0.3723814785480499], "cos_innovation_negg": [0.3799830973148346, 0.47136107087135315, 0.710888147354126], "cos_apical_negg": [0.06476972252130508, 0.24740000069141388, 0.3723814785480499], "cos_Ac_negg": [0.37988266348838806, 0.4748282730579376, 0.7110286951065063], "r_norm": [3.525418281555176, 3.535935878753662, 3.526451826095581]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9505, "test_loss": 0.2087245460510254}, {"step": 2900, "epoch": 6, "train_loss": 0.11933334916830063, "cos_r_negg": [0.060522228479385376, 0.16942700743675232, 0.519522488117218], "cos_innovation_negg": [0.37501612305641174, 0.42948880791664124, 0.7528126239776611], "cos_apical_negg": [0.060522228479385376, 0.16942700743675232, 0.519522488117218], "cos_Ac_negg": [0.37504780292510986, 0.42936092615127563, 0.7527654767036438], "r_norm": [3.549933910369873, 3.558411121368408, 3.5546014308929443]}, {"step": 3000, "epoch": 6, "train_loss": 0.46797239780426025, "cos_r_negg": [0.05155858397483826, 0.24312405288219452, 0.41349536180496216], "cos_innovation_negg": [0.3493667542934418, 0.4458984136581421, 0.662284255027771], "cos_apical_negg": [0.05155858397483826, 0.24312405288219452, 0.41349536180496216], "cos_Ac_negg": [0.3494006097316742, 0.4455445110797882, 0.6621168851852417], "r_norm": [3.533210277557373, 3.5433437824249268, 3.5356082916259766], "ldr": {"dL_sdil": -0.012944579124450684, "dL_bp": -0.01592540740966797, "ratio": 0.812825618300497}}, {"step": 3100, "epoch": 6, "train_loss": 0.22827595472335815, "cos_r_negg": [0.04552757740020752, 0.19341063499450684, 0.2632794678211212], "cos_innovation_negg": [0.26925477385520935, 0.36854177713394165, 0.6529192924499512], "cos_apical_negg": [0.04552757740020752, 0.19341063499450684, 0.2632794678211212], "cos_Ac_negg": [0.26958921551704407, 0.36841315031051636, 0.6540975570678711], "r_norm": [3.562100887298584, 3.5745458602905273, 3.5627670288085938]}, {"step": 3200, "epoch": 6, "train_loss": 0.2890065312385559, "cos_r_negg": [0.07927241921424866, 0.21649539470672607, 0.41718924045562744], "cos_innovation_negg": [0.3627319633960724, 0.49093931913375854, 0.7118546962738037], "cos_apical_negg": [0.07927241921424866, 0.21649539470672607, 0.41718924045562744], "cos_Ac_negg": [0.3627040982246399, 0.49099481105804443, 0.7110843658447266], "r_norm": [3.5166826248168945, 3.52805233001709, 3.523509979248047]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9506, "test_loss": 0.20884917068481446}, {"step": 3300, "epoch": 7, "train_loss": 0.07418417930603027, "cos_r_negg": [0.06989209353923798, 0.12369517982006073, 0.3734971284866333], "cos_innovation_negg": [0.38426846265792847, 0.46338215470314026, 0.5919349193572998], "cos_apical_negg": [0.06989209353923798, 0.12369517982006073, 0.3734971284866333], "cos_Ac_negg": [0.3843148946762085, 0.46331679821014404, 0.5915554761886597], "r_norm": [3.510678291320801, 3.526305675506592, 3.5156121253967285]}, {"step": 3400, "epoch": 7, "train_loss": 0.25392860174179077, "cos_r_negg": [0.04078827053308487, 0.15718908607959747, 0.3688907027244568], "cos_innovation_negg": [0.3101622462272644, 0.3867226243019104, 0.6405729055404663], "cos_apical_negg": [0.04078827053308487, 0.15718908607959747, 0.3688907027244568], "cos_Ac_negg": [0.3103155493736267, 0.3865430951118469, 0.6404472589492798], "r_norm": [3.5139670372009277, 3.5295913219451904, 3.5230026245117188]}, {"step": 3500, "epoch": 7, "train_loss": 0.11625717580318451, "cos_r_negg": [0.037048958241939545, 0.21172501146793365, 0.4897260069847107], "cos_innovation_negg": [0.31466615200042725, 0.3850085139274597, 0.7382959127426147], "cos_apical_negg": [0.037048958241939545, 0.21172501146793365, 0.4897260069847107], "cos_Ac_negg": [0.31472235918045044, 0.38480842113494873, 0.7382181882858276], "r_norm": [3.522158145904541, 3.5345427989959717, 3.525588274002075], "ldr": {"dL_sdil": -0.006321340799331665, "dL_bp": -0.010596394538879395, "ratio": 0.596555816805229}}, {"step": 3600, "epoch": 7, "train_loss": 0.07584749907255173, "cos_r_negg": [0.045071009546518326, 0.1780015379190445, 0.39236947894096375], "cos_innovation_negg": [0.3479326069355011, 0.38996952772140503, 0.6829699277877808], "cos_apical_negg": [0.045071009546518326, 0.1780015379190445, 0.39236947894096375], "cos_Ac_negg": [0.3478541970252991, 0.3900073766708374, 0.6829819679260254], "r_norm": [3.493873357772827, 3.5034313201904297, 3.4956436157226562]}, {"step": 3700, "epoch": 7, "train_loss": 0.11418284475803375, "cos_r_negg": [0.07023665308952332, 0.15918795764446259, 0.4905637502670288], "cos_innovation_negg": [0.39593905210494995, 0.4115228056907654, 0.673789918422699], "cos_apical_negg": [0.07023665308952332, 0.15918795764446259, 0.4905637502670288], "cos_Ac_negg": [0.3959917426109314, 0.4117951989173889, 0.6735410690307617], "r_norm": [3.5024595260620117, 3.5144505500793457, 3.506063938140869]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9484, "test_loss": 0.23533484725952147}, {"step": 3800, "epoch": 8, "train_loss": 0.1606840193271637, "cos_r_negg": [0.05866185575723648, 0.12872251868247986, 0.4020008444786072], "cos_innovation_negg": [0.3276687264442444, 0.39441102743148804, 0.5848279595375061], "cos_apical_negg": [0.05866185575723648, 0.12872251868247986, 0.4020008444786072], "cos_Ac_negg": [0.32772424817085266, 0.39426061511039734, 0.5845370888710022], "r_norm": [3.482597827911377, 3.494558811187744, 3.487753391265869]}, {"step": 3900, "epoch": 8, "train_loss": 0.25916221737861633, "cos_r_negg": [0.050195276737213135, 0.16606280207633972, 0.3071008622646332], "cos_innovation_negg": [0.291739284992218, 0.3520588278770447, 0.6213912963867188], "cos_apical_negg": [0.050195276737213135, 0.16606280207633972, 0.3071008622646332], "cos_Ac_negg": [0.2914842665195465, 0.35185563564300537, 0.6211859583854675], "r_norm": [3.5000383853912354, 3.5071334838867188, 3.503175735473633]}, {"step": 4000, "epoch": 8, "train_loss": 0.2509729266166687, "cos_r_negg": [0.06992388516664505, 0.1116754412651062, 0.26220211386680603], "cos_innovation_negg": [0.35568588972091675, 0.44589173793792725, 0.6574065685272217], "cos_apical_negg": [0.06992388516664505, 0.1116754412651062, 0.26220211386680603], "cos_Ac_negg": [0.35574978590011597, 0.44588202238082886, 0.6573203206062317], "r_norm": [3.488227128982544, 3.500368118286133, 3.4904279708862305], "ldr": {"dL_sdil": -0.006662696599960327, "dL_bp": -0.01077413558959961, "ratio": 0.6183973224164638}}, {"step": 4100, "epoch": 8, "train_loss": 0.1663362681865692, "cos_r_negg": [0.028803933411836624, 0.0883977860212326, 0.33949169516563416], "cos_innovation_negg": [0.3531327247619629, 0.44179436564445496, 0.6693330407142639], "cos_apical_negg": [0.028803933411836624, 0.0883977860212326, 0.33949169516563416], "cos_Ac_negg": [0.3532670736312866, 0.4426644444465637, 0.6688066124916077], "r_norm": [3.4893453121185303, 3.498201847076416, 3.494220495223999]}, {"step": 4200, "epoch": 8, "train_loss": 0.28906285762786865, "cos_r_negg": [0.022621016949415207, 0.10095589607954025, 0.36402660608291626], "cos_innovation_negg": [0.3255588710308075, 0.3754814565181732, 0.68536376953125], "cos_apical_negg": [0.022621016949415207, 0.10095589607954025, 0.36402660608291626], "cos_Ac_negg": [0.32601025700569153, 0.37586450576782227, 0.6850257515907288], "r_norm": [3.5021300315856934, 3.5099406242370605, 3.5010807514190674]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.953, "test_loss": 0.205215665435791}, {"step": 4300, "epoch": 9, "train_loss": 0.12148469686508179, "cos_r_negg": [0.06310656666755676, 0.14638730883598328, 0.4602958858013153], "cos_innovation_negg": [0.36672326922416687, 0.44657689332962036, 0.7104050517082214], "cos_apical_negg": [0.06310656666755676, 0.14638730883598328, 0.4602958858013153], "cos_Ac_negg": [0.3664681911468506, 0.44682544469833374, 0.7106057405471802], "r_norm": [3.506289482116699, 3.5161452293395996, 3.505258560180664]}, {"step": 4400, "epoch": 9, "train_loss": 0.08510865271091461, "cos_r_negg": [-0.010272042825818062, 0.15191876888275146, 0.43028974533081055], "cos_innovation_negg": [0.38079124689102173, 0.44991809129714966, 0.7786355018615723], "cos_apical_negg": [-0.010272042825818062, 0.15191876888275146, 0.43028974533081055], "cos_Ac_negg": [0.38079801201820374, 0.4504637122154236, 0.7784952521324158], "r_norm": [3.5009899139404297, 3.510887622833252, 3.5052731037139893]}, {"step": 4500, "epoch": 9, "train_loss": 0.20403647422790527, "cos_r_negg": [0.010687527246773243, 0.13759338855743408, 0.4406067132949829], "cos_innovation_negg": [0.3841218054294586, 0.5035387277603149, 0.6796131730079651], "cos_apical_negg": [0.010687527246773243, 0.13759338855743408, 0.4406067132949829], "cos_Ac_negg": [0.3842289447784424, 0.503507137298584, 0.6796402931213379], "r_norm": [3.513428211212158, 3.529162645339966, 3.512880563735962], "ldr": {"dL_sdil": -0.007929712533950806, "dL_bp": -0.012386322021484375, "ratio": 0.6401991261164152}}, {"step": 4600, "epoch": 9, "train_loss": 0.03242642432451248, "cos_r_negg": [0.0505828782916069, 0.0804450511932373, 0.32763200998306274], "cos_innovation_negg": [0.31487613916397095, 0.4083035886287689, 0.6746851801872253], "cos_apical_negg": [0.0505828782916069, 0.0804450511932373, 0.32763200998306274], "cos_Ac_negg": [0.3151475787162781, 0.408203125, 0.6746005415916443], "r_norm": [3.521483898162842, 3.5280349254608154, 3.5226235389709473]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9548, "test_loss": 0.2061031898498535}, {"step": 4700, "epoch": 10, "train_loss": 0.1115632876753807, "cos_r_negg": [0.02221045456826687, 0.1120276004076004, 0.4373319745063782], "cos_innovation_negg": [0.32638704776763916, 0.36114078760147095, 0.6762726306915283], "cos_apical_negg": [0.02221045456826687, 0.1120276004076004, 0.4373319745063782], "cos_Ac_negg": [0.3263549208641052, 0.3607480227947235, 0.6741930246353149], "r_norm": [3.476983070373535, 3.483590602874756, 3.476677417755127]}, {"step": 4800, "epoch": 10, "train_loss": 0.12115556746721268, "cos_r_negg": [0.034846387803554535, 0.13012909889221191, 0.40665000677108765], "cos_innovation_negg": [0.3161507546901703, 0.3160267770290375, 0.5843727588653564], "cos_apical_negg": [0.034846387803554535, 0.13012909889221191, 0.40665000677108765], "cos_Ac_negg": [0.31633883714675903, 0.31568193435668945, 0.5841403603553772], "r_norm": [3.4819440841674805, 3.4860775470733643, 3.475830078125]}, {"step": 4900, "epoch": 10, "train_loss": 0.1609702706336975, "cos_r_negg": [0.04368805140256882, 0.09769974648952484, 0.3166388273239136], "cos_innovation_negg": [0.29963433742523193, 0.31716933846473694, 0.5456969738006592], "cos_apical_negg": [0.04368805140256882, 0.09769974648952484, 0.3166388273239136], "cos_Ac_negg": [0.2999567985534668, 0.31802740693092346, 0.5459784269332886], "r_norm": [3.5055782794952393, 3.51322603225708, 3.50827693939209]}, {"step": 5000, "epoch": 10, "train_loss": 0.05106005072593689, "cos_r_negg": [0.042463064193725586, 0.1969190239906311, 0.42296677827835083], "cos_innovation_negg": [0.3468455374240875, 0.42660462856292725, 0.673263669013977], "cos_apical_negg": [0.042463064193725586, 0.1969190239906311, 0.42296677827835083], "cos_Ac_negg": [0.34681013226509094, 0.42831796407699585, 0.6732136011123657], "r_norm": [3.488314151763916, 3.4958815574645996, 3.4880306720733643], "ldr": {"dL_sdil": -0.004139572381973267, "dL_bp": -0.007407546043395996, "ratio": 0.5588318125492847}}, {"step": 5100, "epoch": 10, "train_loss": 0.20244255661964417, "cos_r_negg": [0.014465187676250935, 0.1376720666885376, 0.3820728063583374], "cos_innovation_negg": [0.31749844551086426, 0.3807353079319, 0.5672973394393921], "cos_apical_negg": [0.014465187676250935, 0.1376720666885376, 0.3820728063583374], "cos_Ac_negg": [0.31760525703430176, 0.3803553283214569, 0.5672276020050049], "r_norm": [3.5066823959350586, 3.515922784805298, 3.501880407333374]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.955, "test_loss": 0.2094983467102051}, {"step": 5200, "epoch": 11, "train_loss": 0.07901652902364731, "cos_r_negg": [0.050434064120054245, 0.058497946709394455, 0.23812434077262878], "cos_innovation_negg": [0.32730698585510254, 0.3108316659927368, 0.4035617709159851], "cos_apical_negg": [0.050434064120054245, 0.058497946709394455, 0.23812434077262878], "cos_Ac_negg": [0.32737332582473755, 0.3107430338859558, 0.4036167860031128], "r_norm": [3.4832308292388916, 3.4909329414367676, 3.480499744415283]}, {"step": 5300, "epoch": 11, "train_loss": 0.16419199109077454, "cos_r_negg": [0.053285371512174606, 0.1840730607509613, 0.28839346766471863], "cos_innovation_negg": [0.38993483781814575, 0.37072324752807617, 0.5901734828948975], "cos_apical_negg": [0.053285371512174606, 0.1840730607509613, 0.28839346766471863], "cos_Ac_negg": [0.39002013206481934, 0.3705340027809143, 0.5902138352394104], "r_norm": [3.4742956161499023, 3.4803595542907715, 3.4762659072875977]}, {"step": 5400, "epoch": 11, "train_loss": 0.25575199723243713, "cos_r_negg": [0.049340926110744476, 0.1775239109992981, 0.2837269902229309], "cos_innovation_negg": [0.36274003982543945, 0.4060286283493042, 0.5776951909065247], "cos_apical_negg": [0.049340926110744476, 0.1775239109992981, 0.2837269902229309], "cos_Ac_negg": [0.3627276122570038, 0.4060618281364441, 0.5776838064193726], "r_norm": [3.477604389190674, 3.4840965270996094, 3.472667694091797]}, {"step": 5500, "epoch": 11, "train_loss": 0.21913626790046692, "cos_r_negg": [0.07639949023723602, 0.17942078411579132, 0.2658585011959076], "cos_innovation_negg": [0.3854910731315613, 0.42924195528030396, 0.5578122138977051], "cos_apical_negg": [0.07639949023723602, 0.17942078411579132, 0.2658585011959076], "cos_Ac_negg": [0.38568559288978577, 0.434417724609375, 0.557792067527771], "r_norm": [3.4973573684692383, 3.502743721008301, 3.4955036640167236], "ldr": {"dL_sdil": -0.0059844255447387695, "dL_bp": -0.015154421329498291, "ratio": 0.3948963417751889}}, {"step": 5600, "epoch": 11, "train_loss": 0.13204315304756165, "cos_r_negg": [0.004832189530134201, 0.14195498824119568, 0.356207937002182], "cos_innovation_negg": [0.3070482909679413, 0.4355112910270691, 0.6441358327865601], "cos_apical_negg": [0.004832189530134201, 0.14195498824119568, 0.356207937002182], "cos_Ac_negg": [0.30701562762260437, 0.4354711174964905, 0.6438366174697876], "r_norm": [3.484740734100342, 3.490720272064209, 3.4857611656188965]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9541, "test_loss": 0.21454206924438476}, {"step": 5700, "epoch": 12, "train_loss": 0.11386805772781372, "cos_r_negg": [0.0549413338303566, 0.06245000287890434, 0.47099626064300537], "cos_innovation_negg": [0.32851672172546387, 0.30441874265670776, 0.6426862478256226], "cos_apical_negg": [0.0549413338303566, 0.06245000287890434, 0.47099626064300537], "cos_Ac_negg": [0.32895076274871826, 0.29865115880966187, 0.64252769947052], "r_norm": [3.4699504375457764, 3.475128650665283, 3.4729838371276855]}, {"step": 5800, "epoch": 12, "train_loss": 0.07414205372333527, "cos_r_negg": [0.013079151511192322, 0.061160698533058167, 0.4021567404270172], "cos_innovation_negg": [0.3663828372955322, 0.40112730860710144, 0.6730082035064697], "cos_apical_negg": [0.013079151511192322, 0.061160698533058167, 0.4021567404270172], "cos_Ac_negg": [0.36617815494537354, 0.4005953073501587, 0.6726951599121094], "r_norm": [3.4707837104797363, 3.4784164428710938, 3.472170352935791]}, {"step": 5900, "epoch": 12, "train_loss": 0.10111047327518463, "cos_r_negg": [0.02621365152299404, 0.15886923670768738, 0.36656808853149414], "cos_innovation_negg": [0.3404310345649719, 0.3644443154335022, 0.6749333739280701], "cos_apical_negg": [0.02621365152299404, 0.15886923670768738, 0.36656808853149414], "cos_Ac_negg": [0.3408862054347992, 0.3638870120048523, 0.6749204993247986], "r_norm": [3.4871010780334473, 3.493950366973877, 3.4880800247192383]}, {"step": 6000, "epoch": 12, "train_loss": 0.023485127836465836, "cos_r_negg": [0.05445155128836632, 0.007987810298800468, 0.20682509243488312], "cos_innovation_negg": [0.26833176612854004, 0.2997751235961914, 0.4263879656791687], "cos_apical_negg": [0.05445155128836632, 0.007987810298800468, 0.20682509243488312], "cos_Ac_negg": [0.26820188760757446, 0.30016398429870605, 0.4269087314605713], "r_norm": [3.4766125679016113, 3.4831552505493164, 3.475107431411743], "ldr": {"dL_sdil": -0.006880640983581543, "dL_bp": -0.010458588600158691, "ratio": 0.6578938369826633}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9558, "test_loss": 0.2026500846862793}, {"step": 6100, "epoch": 13, "train_loss": 0.08687175065279007, "cos_r_negg": [0.04893213137984276, 0.17724522948265076, 0.38123536109924316], "cos_innovation_negg": [0.35023096203804016, 0.4337785243988037, 0.7616674900054932], "cos_apical_negg": [0.04893213137984276, 0.17724522948265076, 0.38123536109924316], "cos_Ac_negg": [0.3501967191696167, 0.4347092807292938, 0.7617870569229126], "r_norm": [3.4888854026794434, 3.4950647354125977, 3.488955020904541]}, {"step": 6200, "epoch": 13, "train_loss": 0.04847916215658188, "cos_r_negg": [0.042757563292980194, 0.14450564980506897, 0.2226705104112625], "cos_innovation_negg": [0.290037602186203, 0.36447250843048096, 0.5477343797683716], "cos_apical_negg": [0.042757563292980194, 0.14450564980506897, 0.2226705104112625], "cos_Ac_negg": [0.2902388274669647, 0.36432570219039917, 0.5470004081726074], "r_norm": [3.4833076000213623, 3.4895639419555664, 3.4830546379089355]}, {"step": 6300, "epoch": 13, "train_loss": 0.06115848198533058, "cos_r_negg": [0.05876431614160538, 0.15547120571136475, 0.3499096930027008], "cos_innovation_negg": [0.38396957516670227, 0.4307061433792114, 0.7028771638870239], "cos_apical_negg": [0.05876431614160538, 0.15547120571136475, 0.3499096930027008], "cos_Ac_negg": [0.38495269417762756, 0.4317866563796997, 0.7030075788497925], "r_norm": [3.4799933433532715, 3.48433780670166, 3.4775474071502686]}, {"step": 6400, "epoch": 13, "train_loss": 0.0630403682589531, "cos_r_negg": [0.03731595352292061, 0.13848638534545898, 0.2675327658653259], "cos_innovation_negg": [0.3231140375137329, 0.4235890507698059, 0.6113507747650146], "cos_apical_negg": [0.03731595352292061, 0.13848638534545898, 0.2675327658653259], "cos_Ac_negg": [0.32327771186828613, 0.42362403869628906, 0.6113297343254089], "r_norm": [3.4880356788635254, 3.494199275970459, 3.4880480766296387]}, {"step": 6500, "epoch": 13, "train_loss": 0.01672252267599106, "cos_r_negg": [0.032514844089746475, 0.09534486383199692, 0.4149414002895355], "cos_innovation_negg": [0.29357174038887024, 0.3667604625225067, 0.5798873901367188], "cos_apical_negg": [0.032514844089746475, 0.09534486383199692, 0.4149414002895355], "cos_Ac_negg": [0.29363441467285156, 0.3668266236782074, 0.5797698497772217], "r_norm": [3.474494457244873, 3.4805514812469482, 3.472459554672241], "ldr": {"dL_sdil": -0.005686312913894653, "dL_bp": -0.010570958256721497, "ratio": 0.5379183964026191}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9564, "test_loss": 0.20574900283813477}, {"step": 6600, "epoch": 14, "train_loss": 0.11292517185211182, "cos_r_negg": [0.01754484325647354, 0.11215884983539581, 0.37554875016212463], "cos_innovation_negg": [0.30893057584762573, 0.47593510150909424, 0.6779210567474365], "cos_apical_negg": [0.01754484325647354, 0.11215884983539581, 0.37554875016212463], "cos_Ac_negg": [0.3088195323944092, 0.47575485706329346, 0.6779165863990784], "r_norm": [3.4791736602783203, 3.484938383102417, 3.4762465953826904]}, {"step": 6700, "epoch": 14, "train_loss": 0.0758395716547966, "cos_r_negg": [0.0053497059270739555, 0.15712696313858032, 0.28254202008247375], "cos_innovation_negg": [0.2489399015903473, 0.36492636799812317, 0.6058889031410217], "cos_apical_negg": [0.0053497059270739555, 0.15712696313858032, 0.28254202008247375], "cos_Ac_negg": [0.24940410256385803, 0.3639839291572571, 0.6067221164703369], "r_norm": [3.484689474105835, 3.489269256591797, 3.4857072830200195]}, {"step": 6800, "epoch": 14, "train_loss": 0.13973073661327362, "cos_r_negg": [0.028950165957212448, 0.1441994458436966, 0.3252054750919342], "cos_innovation_negg": [0.2828671336174011, 0.3844386041164398, 0.6908804178237915], "cos_apical_negg": [0.028950165957212448, 0.1441994458436966, 0.3252054750919342], "cos_Ac_negg": [0.2828330993652344, 0.3840353488922119, 0.6907874345779419], "r_norm": [3.4923133850097656, 3.496694564819336, 3.492377281188965]}, {"step": 6900, "epoch": 14, "train_loss": 0.11943142861127853, "cos_r_negg": [0.03463827446103096, 0.15241678059101105, 0.30104637145996094], "cos_innovation_negg": [0.2674281895160675, 0.3884856104850769, 0.6272569894790649], "cos_apical_negg": [0.03463827446103096, 0.15241678059101105, 0.30104637145996094], "cos_Ac_negg": [0.26769256591796875, 0.3886655569076538, 0.6270128488540649], "r_norm": [3.4798500537872314, 3.4845900535583496, 3.4785399436950684]}, {"step": 7000, "epoch": 14, "train_loss": 0.07303012907505035, "cos_r_negg": [0.027918649837374687, 0.11706951260566711, 0.15363363921642303], "cos_innovation_negg": [0.2829412817955017, 0.37086641788482666, 0.6008607149124146], "cos_apical_negg": [0.027918649837374687, 0.11706951260566711, 0.15363363921642303], "cos_Ac_negg": [0.28335750102996826, 0.3724294900894165, 0.6002528667449951], "r_norm": [3.4688148498535156, 3.471724510192871, 3.464806079864502], "ldr": {"dL_sdil": -0.0035640597343444824, "dL_bp": -0.008194401860237122, "ratio": 0.4349383634257533}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9565, "test_loss": 0.21070952911376953}], "final": {"test_acc": 0.9565, "test_loss": 0.21070952911376953, "wall_s": 23.235268354415894, "cos_r_negg": [0.03801558166742325, 0.08539733290672302, 0.2681865394115448], "cos_innovation_negg": [0.2545422315597534, 0.3348400890827179, 0.7059264183044434], "cos_apical_negg": [0.03801558166742325, 0.08539733290672302, 0.2681865394115448], "cos_Ac_negg": [0.2547372281551361, 0.3345704674720764, 0.7062969207763672]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_raw_s3.json b/results/nuis_ctrl_v1_rho0p2_raw_s3.json new file mode 100644 index 0000000..9238ca1 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_raw_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_raw_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.021260136738419533, 0.012223788537085056, 0.0022221505641937256], "cos_innovation_negg": [-0.019810983911156654, 0.015065358020365238, 0.006113413721323013], "cos_apical_negg": [-0.021260136738419533, 0.012223788537085056, 0.0022221505641937256], "cos_Ac_negg": [-0.019811661913990974, 0.015065068379044533, 0.006112872157245874], "r_norm": [5.542547225952148, 5.094986915588379, 5.096925735473633], "ldr": {"dL_sdil": 0.02908468246459961, "dL_bp": -0.21986818313598633, "ratio": -0.13228236141202393}}, {"step": 100, "epoch": 0, "train_loss": 0.4162105321884155, "cos_r_negg": [0.15874551236629486, 0.22092744708061218, 0.5825601816177368], "cos_innovation_negg": [0.33813774585723877, 0.36902719736099243, 0.6371220350265503], "cos_apical_negg": [0.15874551236629486, 0.22092744708061218, 0.5825601816177368], "cos_Ac_negg": [0.33794471621513367, 0.36896833777427673, 0.6371626257896423], "r_norm": [3.947782039642334, 3.913179397583008, 3.8808839321136475]}, {"step": 200, "epoch": 0, "train_loss": 0.40432149171829224, "cos_r_negg": [0.1312902271747589, 0.24083729088306427, 0.6597644090652466], "cos_innovation_negg": [0.37063735723495483, 0.41428452730178833, 0.7216600179672241], "cos_apical_negg": [0.1312902271747589, 0.24083729088306427, 0.6597644090652466], "cos_Ac_negg": [0.37025684118270874, 0.4141829013824463, 0.721627950668335], "r_norm": [3.757382869720459, 3.742218494415283, 3.7302162647247314]}, {"step": 300, "epoch": 0, "train_loss": 0.42480331659317017, "cos_r_negg": [0.1358795166015625, 0.18862777948379517, 0.5736256241798401], "cos_innovation_negg": [0.40564265847206116, 0.44770586490631104, 0.716882586479187], "cos_apical_negg": [0.1358795166015625, 0.18862777948379517, 0.5736256241798401], "cos_Ac_negg": [0.4055516719818115, 0.4467802345752716, 0.716884195804596], "r_norm": [3.6736836433410645, 3.6733498573303223, 3.658867835998535]}, {"step": 400, "epoch": 0, "train_loss": 0.3001886010169983, "cos_r_negg": [0.09697838872671127, 0.1948096603155136, 0.6268853545188904], "cos_innovation_negg": [0.4034644365310669, 0.47457218170166016, 0.7403368949890137], "cos_apical_negg": [0.09697838872671127, 0.1948096603155136, 0.6268853545188904], "cos_Ac_negg": [0.4033777713775635, 0.4742700457572937, 0.7401320934295654], "r_norm": [3.652329683303833, 3.6502227783203125, 3.6350271701812744]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9223, "test_loss": 0.3095009246826172}, {"step": 500, "epoch": 1, "train_loss": 0.29689911007881165, "cos_r_negg": [0.10997963696718216, 0.15761850774288177, 0.48379814624786377], "cos_innovation_negg": [0.3912685513496399, 0.4725545048713684, 0.680726170539856], "cos_apical_negg": [0.10997963696718216, 0.15761850774288177, 0.48379814624786377], "cos_Ac_negg": [0.39121493697166443, 0.4711601734161377, 0.6806540489196777], "r_norm": [3.6223111152648926, 3.6209604740142822, 3.603039503097534], "ldr": {"dL_sdil": -0.01956579089164734, "dL_bp": -0.0209214985370636, "ratio": 0.9352002609653153}}, {"step": 600, "epoch": 1, "train_loss": 0.16214382648468018, "cos_r_negg": [0.11992692947387695, 0.19323208928108215, 0.4143443703651428], "cos_innovation_negg": [0.45397698879241943, 0.48377689719200134, 0.6378847360610962], "cos_apical_negg": [0.11992692947387695, 0.19323208928108215, 0.4143443703651428], "cos_Ac_negg": [0.45380228757858276, 0.481845498085022, 0.6378047466278076], "r_norm": [3.6181607246398926, 3.6285390853881836, 3.6141698360443115]}, {"step": 700, "epoch": 1, "train_loss": 0.32526838779449463, "cos_r_negg": [0.1002906858921051, 0.17312973737716675, 0.4733106791973114], "cos_innovation_negg": [0.39862507581710815, 0.46217551827430725, 0.6866208910942078], "cos_apical_negg": [0.1002906858921051, 0.17312973737716675, 0.4733106791973114], "cos_Ac_negg": [0.3985694944858551, 0.4606516659259796, 0.6863213777542114], "r_norm": [3.57474422454834, 3.5901570320129395, 3.5710339546203613]}, {"step": 800, "epoch": 1, "train_loss": 0.2385207712650299, "cos_r_negg": [0.1118660643696785, 0.17899373173713684, 0.444225549697876], "cos_innovation_negg": [0.38075879216194153, 0.44652023911476135, 0.6085976362228394], "cos_apical_negg": [0.1118660643696785, 0.17899373173713684, 0.444225549697876], "cos_Ac_negg": [0.3806275427341461, 0.446380078792572, 0.6075937151908875], "r_norm": [3.605712413787842, 3.611281156539917, 3.592348575592041]}, {"step": 900, "epoch": 1, "train_loss": 0.35703274607658386, "cos_r_negg": [0.11123300343751907, 0.19251804053783417, 0.5006859302520752], "cos_innovation_negg": [0.44401121139526367, 0.39035892486572266, 0.7030285596847534], "cos_apical_negg": [0.11123300343751907, 0.19251804053783417, 0.5006859302520752], "cos_Ac_negg": [0.443547785282135, 0.38969066739082336, 0.7024809122085571], "r_norm": [3.5787973403930664, 3.59369158744812, 3.5637035369873047]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9381, "test_loss": 0.25364563827514647}, {"step": 1000, "epoch": 2, "train_loss": 0.10202008485794067, "cos_r_negg": [0.07427409291267395, 0.14725729823112488, 0.3841191530227661], "cos_innovation_negg": [0.42140308022499084, 0.4790295958518982, 0.7235931158065796], "cos_apical_negg": [0.07427409291267395, 0.14725729823112488, 0.3841191530227661], "cos_Ac_negg": [0.4214572310447693, 0.47863540053367615, 0.722671389579773], "r_norm": [3.5861001014709473, 3.591590166091919, 3.5841143131256104], "ldr": {"dL_sdil": -0.012681841850280762, "dL_bp": -0.01486918330192566, "ratio": 0.8528943111918177}}, {"step": 1100, "epoch": 2, "train_loss": 0.37036916613578796, "cos_r_negg": [0.07001405954360962, 0.18505609035491943, 0.6536509394645691], "cos_innovation_negg": [0.40204501152038574, 0.46515873074531555, 0.7121667861938477], "cos_apical_negg": [0.07001405954360962, 0.18505609035491943, 0.6536509394645691], "cos_Ac_negg": [0.4020274877548218, 0.4648301303386688, 0.711942195892334], "r_norm": [3.5514001846313477, 3.5586211681365967, 3.545408248901367]}, {"step": 1200, "epoch": 2, "train_loss": 0.24266694486141205, "cos_r_negg": [0.05192887410521507, 0.15708720684051514, 0.5694223642349243], "cos_innovation_negg": [0.38805338740348816, 0.45369160175323486, 0.6710066795349121], "cos_apical_negg": [0.05192887410521507, 0.15708720684051514, 0.5694223642349243], "cos_Ac_negg": [0.3879629671573639, 0.453529953956604, 0.670644998550415], "r_norm": [3.551464080810547, 3.5580592155456543, 3.5389885902404785]}, {"step": 1300, "epoch": 2, "train_loss": 0.2787606418132782, "cos_r_negg": [0.11774163693189621, 0.20745837688446045, 0.5347984433174133], "cos_innovation_negg": [0.4150112569332123, 0.415810763835907, 0.7518109083175659], "cos_apical_negg": [0.11774163693189621, 0.20745837688446045, 0.5347984433174133], "cos_Ac_negg": [0.4146523177623749, 0.41517186164855957, 0.7517269849777222], "r_norm": [3.5791049003601074, 3.582185745239258, 3.5554018020629883]}, {"step": 1400, "epoch": 2, "train_loss": 0.2832641005516052, "cos_r_negg": [0.06006778031587601, 0.1871626079082489, 0.5029486417770386], "cos_innovation_negg": [0.38534796237945557, 0.4367636442184448, 0.7292816042900085], "cos_apical_negg": [0.06006778031587601, 0.1871626079082489, 0.5029486417770386], "cos_Ac_negg": [0.3854367733001709, 0.43669161200523376, 0.7291083931922913], "r_norm": [3.5427424907684326, 3.548611879348755, 3.537234306335449]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.94, "test_loss": 0.2331307273864746}, {"step": 1500, "epoch": 3, "train_loss": 0.18998271226882935, "cos_r_negg": [0.059603121131658554, 0.21340185403823853, 0.4743293821811676], "cos_innovation_negg": [0.4057149589061737, 0.4672331213951111, 0.6827440857887268], "cos_apical_negg": [0.059603121131658554, 0.21340185403823853, 0.4743293821811676], "cos_Ac_negg": [0.40591418743133545, 0.4667016267776489, 0.6824369430541992], "r_norm": [3.5439181327819824, 3.551581382751465, 3.529449462890625], "ldr": {"dL_sdil": -0.012014731764793396, "dL_bp": -0.016467899084091187, "ratio": 0.7295849764102713}}, {"step": 1600, "epoch": 3, "train_loss": 0.1765434741973877, "cos_r_negg": [0.0973219946026802, 0.16097596287727356, 0.4674224257469177], "cos_innovation_negg": [0.40727871656417847, 0.4308268427848816, 0.649423360824585], "cos_apical_negg": [0.0973219946026802, 0.16097596287727356, 0.4674224257469177], "cos_Ac_negg": [0.4073278307914734, 0.43071120977401733, 0.6491749286651611], "r_norm": [3.518907070159912, 3.5303657054901123, 3.524209976196289]}, {"step": 1700, "epoch": 3, "train_loss": 0.17348064482212067, "cos_r_negg": [0.0876544937491417, 0.14669957756996155, 0.44371700286865234], "cos_innovation_negg": [0.4204227924346924, 0.4727793335914612, 0.7431782484054565], "cos_apical_negg": [0.0876544937491417, 0.14669957756996155, 0.44371700286865234], "cos_Ac_negg": [0.4204082190990448, 0.47266829013824463, 0.7429771423339844], "r_norm": [3.5307886600494385, 3.54005765914917, 3.5241928100585938]}, {"step": 1800, "epoch": 3, "train_loss": 0.0810948982834816, "cos_r_negg": [0.06658409535884857, 0.16938799619674683, 0.3872460126876831], "cos_innovation_negg": [0.3868839144706726, 0.4324009418487549, 0.6475657224655151], "cos_apical_negg": [0.06658409535884857, 0.16938799619674683, 0.3872460126876831], "cos_Ac_negg": [0.3865867555141449, 0.43152835965156555, 0.6473391056060791], "r_norm": [3.543121337890625, 3.5587081909179688, 3.5440807342529297]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9429, "test_loss": 0.23492875289916992}, {"step": 1900, "epoch": 4, "train_loss": 0.16232708096504211, "cos_r_negg": [0.04799919202923775, 0.17479977011680603, 0.5038758516311646], "cos_innovation_negg": [0.3665763735771179, 0.3608493208885193, 0.5980464816093445], "cos_apical_negg": [0.04799919202923775, 0.17479977011680603, 0.5038758516311646], "cos_Ac_negg": [0.366635262966156, 0.3607875108718872, 0.5979119539260864], "r_norm": [3.5108537673950195, 3.5219154357910156, 3.5041284561157227]}, {"step": 2000, "epoch": 4, "train_loss": 0.24531340599060059, "cos_r_negg": [0.04944268614053726, 0.1547052562236786, 0.3992457389831543], "cos_innovation_negg": [0.34468215703964233, 0.3923376798629761, 0.6807520389556885], "cos_apical_negg": [0.04944268614053726, 0.1547052562236786, 0.3992457389831543], "cos_Ac_negg": [0.344568133354187, 0.3918842077255249, 0.6803131699562073], "r_norm": [3.502753973007202, 3.5167713165283203, 3.5009493827819824], "ldr": {"dL_sdil": -0.008263975381851196, "dL_bp": -0.011206656694412231, "ratio": 0.7374166628992668}}, {"step": 2100, "epoch": 4, "train_loss": 0.09542512893676758, "cos_r_negg": [0.08087408542633057, 0.20022933185100555, 0.4512237310409546], "cos_innovation_negg": [0.3535239100456238, 0.44054579734802246, 0.6819437146186829], "cos_apical_negg": [0.08087408542633057, 0.20022933185100555, 0.4512237310409546], "cos_Ac_negg": [0.3536118268966675, 0.4402439296245575, 0.6818094253540039], "r_norm": [3.5047695636749268, 3.52346134185791, 3.4986732006073]}, {"step": 2200, "epoch": 4, "train_loss": 0.09431038051843643, "cos_r_negg": [0.05138937383890152, 0.1438717544078827, 0.3813856244087219], "cos_innovation_negg": [0.33028802275657654, 0.4082421660423279, 0.6206116080284119], "cos_apical_negg": [0.05138937383890152, 0.1438717544078827, 0.3813856244087219], "cos_Ac_negg": [0.3305448889732361, 0.408416211605072, 0.6204746961593628], "r_norm": [3.4864883422851562, 3.498483896255493, 3.4766502380371094]}, {"step": 2300, "epoch": 4, "train_loss": 0.4217609763145447, "cos_r_negg": [0.058337435126304626, 0.1560928374528885, 0.4097459018230438], "cos_innovation_negg": [0.36207762360572815, 0.36189624667167664, 0.6322119235992432], "cos_apical_negg": [0.058337435126304626, 0.1560928374528885, 0.4097459018230438], "cos_Ac_negg": [0.362271785736084, 0.36120668053627014, 0.631930947303772], "r_norm": [3.4944682121276855, 3.501337766647339, 3.494312286376953]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9466, "test_loss": 0.23418847351074218}, {"step": 2400, "epoch": 5, "train_loss": 0.23749515414237976, "cos_r_negg": [0.0601377934217453, 0.08194379508495331, 0.34273505210876465], "cos_innovation_negg": [0.35645896196365356, 0.37637460231781006, 0.6510698795318604], "cos_apical_negg": [0.0601377934217453, 0.08194379508495331, 0.34273505210876465], "cos_Ac_negg": [0.35624855756759644, 0.37726250290870667, 0.6506269574165344], "r_norm": [3.4991185665130615, 3.5102615356445312, 3.500972270965576]}, {"step": 2500, "epoch": 5, "train_loss": 0.1582573503255844, "cos_r_negg": [0.049548257142305374, 0.17181652784347534, 0.4169296622276306], "cos_innovation_negg": [0.40471115708351135, 0.4258463382720947, 0.6607975959777832], "cos_apical_negg": [0.049548257142305374, 0.17181652784347534, 0.4169296622276306], "cos_Ac_negg": [0.4047572612762451, 0.42745718359947205, 0.6607573628425598], "r_norm": [3.479994535446167, 3.4908289909362793, 3.4745736122131348], "ldr": {"dL_sdil": -0.004615426063537598, "dL_bp": -0.007322132587432861, "ratio": 0.6303390451381823}}, {"step": 2600, "epoch": 5, "train_loss": 0.1281326711177826, "cos_r_negg": [0.06857150048017502, 0.1230306327342987, 0.44765129685401917], "cos_innovation_negg": [0.35401198267936707, 0.4161933958530426, 0.6825858354568481], "cos_apical_negg": [0.06857150048017502, 0.1230306327342987, 0.44765129685401917], "cos_Ac_negg": [0.3542211651802063, 0.41626468300819397, 0.6818445920944214], "r_norm": [3.5069639682769775, 3.507291793823242, 3.493438959121704]}, {"step": 2700, "epoch": 5, "train_loss": 0.1660367250442505, "cos_r_negg": [0.06840892136096954, 0.1563005894422531, 0.3342798054218292], "cos_innovation_negg": [0.33757269382476807, 0.4192643165588379, 0.7599515914916992], "cos_apical_negg": [0.06840892136096954, 0.1563005894422531, 0.3342798054218292], "cos_Ac_negg": [0.3375319242477417, 0.41915011405944824, 0.7599398493766785], "r_norm": [3.494847536087036, 3.4929122924804688, 3.4772047996520996]}, {"step": 2800, "epoch": 5, "train_loss": 0.24149787425994873, "cos_r_negg": [0.07307302951812744, 0.16137751936912537, 0.36761873960494995], "cos_innovation_negg": [0.38132530450820923, 0.4052523374557495, 0.7386304140090942], "cos_apical_negg": [0.07307302951812744, 0.16137751936912537, 0.36761873960494995], "cos_Ac_negg": [0.38105469942092896, 0.40415656566619873, 0.7386842370033264], "r_norm": [3.449674367904663, 3.463880777359009, 3.451437473297119]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9513, "test_loss": 0.2174478500366211}, {"step": 2900, "epoch": 6, "train_loss": 0.07711939513683319, "cos_r_negg": [0.04050252586603165, 0.1426362693309784, 0.3001578450202942], "cos_innovation_negg": [0.3493475615978241, 0.3952561020851135, 0.7318971157073975], "cos_apical_negg": [0.04050252586603165, 0.1426362693309784, 0.3001578450202942], "cos_Ac_negg": [0.34944596886634827, 0.39486464858055115, 0.7316437363624573], "r_norm": [3.482269287109375, 3.486943006515503, 3.4676666259765625]}, {"step": 3000, "epoch": 6, "train_loss": 0.42407917976379395, "cos_r_negg": [0.04940447583794594, 0.11705606430768967, 0.5323392748832703], "cos_innovation_negg": [0.3132840096950531, 0.3791601061820984, 0.6816052198410034], "cos_apical_negg": [0.04940447583794594, 0.11705606430768967, 0.5323392748832703], "cos_Ac_negg": [0.31325966119766235, 0.3786467909812927, 0.6814217567443848], "r_norm": [3.4624409675598145, 3.476898670196533, 3.4619574546813965], "ldr": {"dL_sdil": -0.005963370203971863, "dL_bp": -0.00815679132938385, "ratio": 0.7310926518972658}}, {"step": 3100, "epoch": 6, "train_loss": 0.19019417464733124, "cos_r_negg": [0.07474444061517715, 0.15391410887241364, 0.3995736837387085], "cos_innovation_negg": [0.40633130073547363, 0.44038987159729004, 0.5836098194122314], "cos_apical_negg": [0.07474444061517715, 0.15391410887241364, 0.3995736837387085], "cos_Ac_negg": [0.4064522087574005, 0.4404410123825073, 0.583100438117981], "r_norm": [3.493776321411133, 3.498018264770508, 3.48447322845459]}, {"step": 3200, "epoch": 6, "train_loss": 0.34381577372550964, "cos_r_negg": [0.03660139441490173, 0.14431676268577576, 0.461452454328537], "cos_innovation_negg": [0.32411450147628784, 0.35899871587753296, 0.5735756754875183], "cos_apical_negg": [0.03660139441490173, 0.14431676268577576, 0.461452454328537], "cos_Ac_negg": [0.3241834044456482, 0.3587384819984436, 0.5719325542449951], "r_norm": [3.4597373008728027, 3.471553325653076, 3.4588279724121094]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9519, "test_loss": 0.20405164794921876}, {"step": 3300, "epoch": 7, "train_loss": 0.1176857203245163, "cos_r_negg": [0.061337538063526154, 0.13889089226722717, 0.29329049587249756], "cos_innovation_negg": [0.37684375047683716, 0.35231858491897583, 0.7848672270774841], "cos_apical_negg": [0.061337538063526154, 0.13889089226722717, 0.29329049587249756], "cos_Ac_negg": [0.3768061101436615, 0.3547629714012146, 0.7833143472671509], "r_norm": [3.4795916080474854, 3.488805055618286, 3.4705424308776855]}, {"step": 3400, "epoch": 7, "train_loss": 0.2725895345211029, "cos_r_negg": [0.08176343142986298, 0.14919818937778473, 0.40787607431411743], "cos_innovation_negg": [0.36064159870147705, 0.40586963295936584, 0.8122523427009583], "cos_apical_negg": [0.08176343142986298, 0.14919818937778473, 0.40787607431411743], "cos_Ac_negg": [0.36077794432640076, 0.40540987253189087, 0.8120865225791931], "r_norm": [3.4728806018829346, 3.480339527130127, 3.4693703651428223]}, {"step": 3500, "epoch": 7, "train_loss": 0.17209014296531677, "cos_r_negg": [0.034506890922784805, 0.1794096678495407, 0.382910817861557], "cos_innovation_negg": [0.3424011468887329, 0.41062992811203003, 0.7682549357414246], "cos_apical_negg": [0.034506890922784805, 0.1794096678495407, 0.382910817861557], "cos_Ac_negg": [0.3410038352012634, 0.4104558229446411, 0.7682271003723145], "r_norm": [3.450254201889038, 3.4585351943969727, 3.442410469055176], "ldr": {"dL_sdil": -0.004549816250801086, "dL_bp": -0.006936892867088318, "ratio": 0.6558867691884681}}, {"step": 3600, "epoch": 7, "train_loss": 0.06483013927936554, "cos_r_negg": [-0.0054534683004021645, 0.08901366591453552, 0.4238293170928955], "cos_innovation_negg": [0.341597318649292, 0.3883242607116699, 0.7238288521766663], "cos_apical_negg": [-0.0054534683004021645, 0.08901366591453552, 0.4238293170928955], "cos_Ac_negg": [0.3415205478668213, 0.3897556662559509, 0.7235661745071411], "r_norm": [3.4566779136657715, 3.465440511703491, 3.453073024749756]}, {"step": 3700, "epoch": 7, "train_loss": 0.09281452000141144, "cos_r_negg": [0.03211240842938423, 0.13545700907707214, 0.24162913858890533], "cos_innovation_negg": [0.3363017439842224, 0.3803873658180237, 0.6417583227157593], "cos_apical_negg": [0.03211240842938423, 0.13545700907707214, 0.24162913858890533], "cos_Ac_negg": [0.3363775610923767, 0.3809160888195038, 0.6418699622154236], "r_norm": [3.4565465450286865, 3.4678473472595215, 3.4519925117492676]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.954, "test_loss": 0.19522984848022462}, {"step": 3800, "epoch": 8, "train_loss": 0.18323449790477753, "cos_r_negg": [0.035001564770936966, 0.10666622221469879, 0.3764757513999939], "cos_innovation_negg": [0.38499557971954346, 0.39564603567123413, 0.6121604442596436], "cos_apical_negg": [0.035001564770936966, 0.10666622221469879, 0.3764757513999939], "cos_Ac_negg": [0.3852468430995941, 0.39480066299438477, 0.6119060516357422], "r_norm": [3.4411020278930664, 3.4525816440582275, 3.442863702774048]}, {"step": 3900, "epoch": 8, "train_loss": 0.25970011949539185, "cos_r_negg": [0.015320001170039177, 0.09205085039138794, 0.32433533668518066], "cos_innovation_negg": [0.3219333589076996, 0.3815285563468933, 0.7037261724472046], "cos_apical_negg": [0.015320001170039177, 0.09205085039138794, 0.32433533668518066], "cos_Ac_negg": [0.3228248059749603, 0.3813602924346924, 0.7039339542388916], "r_norm": [3.460613250732422, 3.47074031829834, 3.45516300201416]}, {"step": 4000, "epoch": 8, "train_loss": 0.17258679866790771, "cos_r_negg": [0.06727051734924316, 0.1155916377902031, 0.3965470790863037], "cos_innovation_negg": [0.35004961490631104, 0.37019461393356323, 0.796610951423645], "cos_apical_negg": [0.06727051734924316, 0.1155916377902031, 0.3965470790863037], "cos_Ac_negg": [0.3499543070793152, 0.3705882728099823, 0.796570897102356], "r_norm": [3.4727070331573486, 3.4746365547180176, 3.4626946449279785], "ldr": {"dL_sdil": -0.006612107157707214, "dL_bp": -0.010546699166297913, "ratio": 0.6269361677477511}}, {"step": 4100, "epoch": 8, "train_loss": 0.06699150800704956, "cos_r_negg": [0.03049563616514206, 0.06994563341140747, 0.18906044960021973], "cos_innovation_negg": [0.301862895488739, 0.3497464060783386, 0.5386556386947632], "cos_apical_negg": [0.03049563616514206, 0.06994563341140747, 0.18906044960021973], "cos_Ac_negg": [0.30170467495918274, 0.34957149624824524, 0.5389940142631531], "r_norm": [3.4565234184265137, 3.463350296020508, 3.4477415084838867]}, {"step": 4200, "epoch": 8, "train_loss": 0.23916780948638916, "cos_r_negg": [0.03248034417629242, 0.09815968573093414, 0.24876968562602997], "cos_innovation_negg": [0.2904040217399597, 0.35109078884124756, 0.7257757782936096], "cos_apical_negg": [0.03248034417629242, 0.09815968573093414, 0.24876968562602997], "cos_Ac_negg": [0.29022103548049927, 0.35067451000213623, 0.7260218858718872], "r_norm": [3.477200984954834, 3.480375289916992, 3.458221197128296]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9528, "test_loss": 0.19846216583251952}, {"step": 4300, "epoch": 9, "train_loss": 0.12377635389566422, "cos_r_negg": [0.029537171125411987, 0.11402608454227448, 0.4001530110836029], "cos_innovation_negg": [0.30419397354125977, 0.35219693183898926, 0.7891754508018494], "cos_apical_negg": [0.029537171125411987, 0.11402608454227448, 0.4001530110836029], "cos_Ac_negg": [0.30461567640304565, 0.35133665800094604, 0.7890763282775879], "r_norm": [3.466557025909424, 3.4750115871429443, 3.4541056156158447]}, {"step": 4400, "epoch": 9, "train_loss": 0.0679711103439331, "cos_r_negg": [0.05254673212766647, 0.14841118454933167, 0.25061288475990295], "cos_innovation_negg": [0.3145867586135864, 0.38231945037841797, 0.6894621253013611], "cos_apical_negg": [0.05254673212766647, 0.14841118454933167, 0.25061288475990295], "cos_Ac_negg": [0.31438493728637695, 0.3823874592781067, 0.6896453499794006], "r_norm": [3.453580379486084, 3.4611690044403076, 3.4457221031188965]}, {"step": 4500, "epoch": 9, "train_loss": 0.15776431560516357, "cos_r_negg": [0.039300091564655304, 0.06973789632320404, 0.28350144624710083], "cos_innovation_negg": [0.3218528628349304, 0.3751370310783386, 0.6877709031105042], "cos_apical_negg": [0.039300091564655304, 0.06973789632320404, 0.28350144624710083], "cos_Ac_negg": [0.3219459354877472, 0.37493467330932617, 0.6877540349960327], "r_norm": [3.4776456356048584, 3.475266456604004, 3.457289218902588], "ldr": {"dL_sdil": -0.008198350667953491, "dL_bp": -0.01223115622997284, "ratio": 0.6702841917645669}}, {"step": 4600, "epoch": 9, "train_loss": 0.1098126769065857, "cos_r_negg": [0.06136070191860199, 0.08569829165935516, 0.2817522883415222], "cos_innovation_negg": [0.34091806411743164, 0.3915615379810333, 0.6436882019042969], "cos_apical_negg": [0.06136070191860199, 0.08569829165935516, 0.2817522883415222], "cos_Ac_negg": [0.339865118265152, 0.39034155011177063, 0.643671452999115], "r_norm": [3.459300994873047, 3.4665286540985107, 3.4506759643554688]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9561, "test_loss": 0.1894625659942627}, {"step": 4700, "epoch": 10, "train_loss": 0.15769211947917938, "cos_r_negg": [0.025960199534893036, 0.0587516687810421, 0.3528662919998169], "cos_innovation_negg": [0.28081679344177246, 0.3753991425037384, 0.6638496518135071], "cos_apical_negg": [0.025960199534893036, 0.0587516687810421, 0.3528662919998169], "cos_Ac_negg": [0.28090617060661316, 0.3753526210784912, 0.6638211011886597], "r_norm": [3.4428272247314453, 3.451734781265259, 3.442997932434082]}, {"step": 4800, "epoch": 10, "train_loss": 0.08338607847690582, "cos_r_negg": [0.06638196110725403, 0.07327023893594742, 0.2562228739261627], "cos_innovation_negg": [0.2585318684577942, 0.27654772996902466, 0.6081768870353699], "cos_apical_negg": [0.06638196110725403, 0.07327023893594742, 0.2562228739261627], "cos_Ac_negg": [0.25858789682388306, 0.2767905592918396, 0.6080375909805298], "r_norm": [3.454063892364502, 3.466850519180298, 3.4523420333862305]}, {"step": 4900, "epoch": 10, "train_loss": 0.15375329554080963, "cos_r_negg": [0.04399401322007179, 0.056140750646591187, 0.33442386984825134], "cos_innovation_negg": [0.37793660163879395, 0.40251556038856506, 0.7320525050163269], "cos_apical_negg": [0.04399401322007179, 0.056140750646591187, 0.33442386984825134], "cos_Ac_negg": [0.3780330717563629, 0.4022142291069031, 0.732097864151001], "r_norm": [3.45633602142334, 3.4598231315612793, 3.4472131729125977]}, {"step": 5000, "epoch": 10, "train_loss": 0.04300476983189583, "cos_r_negg": [0.0394136905670166, 0.1122613251209259, 0.37005919218063354], "cos_innovation_negg": [0.295229434967041, 0.3921196162700653, 0.8542808294296265], "cos_apical_negg": [0.0394136905670166, 0.1122613251209259, 0.37005919218063354], "cos_Ac_negg": [0.2950774133205414, 0.3932495713233948, 0.8537659049034119], "r_norm": [3.459404468536377, 3.4651737213134766, 3.449828624725342], "ldr": {"dL_sdil": -0.00439879298210144, "dL_bp": -0.007524967193603516, "ratio": 0.5845597554020657}}, {"step": 5100, "epoch": 10, "train_loss": 0.17405414581298828, "cos_r_negg": [0.033386439085006714, 0.11723385006189346, 0.3706253468990326], "cos_innovation_negg": [0.30274510383605957, 0.38079744577407837, 0.6484284400939941], "cos_apical_negg": [0.033386439085006714, 0.11723385006189346, 0.3706253468990326], "cos_Ac_negg": [0.30267709493637085, 0.3810500502586365, 0.648541271686554], "r_norm": [3.467454433441162, 3.4689958095550537, 3.458220958709717]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9546, "test_loss": 0.21295623321533202}, {"step": 5200, "epoch": 11, "train_loss": 0.09942180663347244, "cos_r_negg": [0.010495748370885849, 0.15319561958312988, 0.371238648891449], "cos_innovation_negg": [0.30131006240844727, 0.35292017459869385, 0.7025488615036011], "cos_apical_negg": [0.010495748370885849, 0.15319561958312988, 0.371238648891449], "cos_Ac_negg": [0.3014470934867859, 0.35280048847198486, 0.7013906836509705], "r_norm": [3.4570045471191406, 3.4597036838531494, 3.447629451751709]}, {"step": 5300, "epoch": 11, "train_loss": 0.14625895023345947, "cos_r_negg": [0.019430043175816536, 0.09450478851795197, 0.29394441843032837], "cos_innovation_negg": [0.34739500284194946, 0.38494861125946045, 0.7247146368026733], "cos_apical_negg": [0.019430043175816536, 0.09450478851795197, 0.29394441843032837], "cos_Ac_negg": [0.3474361300468445, 0.384884238243103, 0.7245177626609802], "r_norm": [3.4390177726745605, 3.4456520080566406, 3.433529853820801]}, {"step": 5400, "epoch": 11, "train_loss": 0.3815159797668457, "cos_r_negg": [0.054415710270404816, 0.14118674397468567, 0.3518367409706116], "cos_innovation_negg": [0.3308723270893097, 0.40393099188804626, 0.7376440763473511], "cos_apical_negg": [0.054415710270404816, 0.14118674397468567, 0.3518367409706116], "cos_Ac_negg": [0.33153006434440613, 0.4033830165863037, 0.7374203205108643], "r_norm": [3.4774248600006104, 3.4845943450927734, 3.4577105045318604]}, {"step": 5500, "epoch": 11, "train_loss": 0.21437236666679382, "cos_r_negg": [0.03780059516429901, 0.10255183279514313, 0.10307836532592773], "cos_innovation_negg": [0.302434504032135, 0.3519688844680786, 0.6594069004058838], "cos_apical_negg": [0.03780059516429901, 0.10255183279514313, 0.10307836532592773], "cos_Ac_negg": [0.30132365226745605, 0.3522098660469055, 0.6598182916641235], "r_norm": [3.4479527473449707, 3.454310894012451, 3.442077875137329], "ldr": {"dL_sdil": -0.003917321562767029, "dL_bp": -0.006406247615814209, "ratio": 0.6114845690786107}}, {"step": 5600, "epoch": 11, "train_loss": 0.12650348246097565, "cos_r_negg": [0.011236418038606644, 0.0739874318242073, 0.3355979323387146], "cos_innovation_negg": [0.36386924982070923, 0.42239415645599365, 0.6597896218299866], "cos_apical_negg": [0.011236418038606644, 0.0739874318242073, 0.3355979323387146], "cos_Ac_negg": [0.36393678188323975, 0.425212025642395, 0.6598107218742371], "r_norm": [3.4677987098693848, 3.4664454460144043, 3.4592888355255127]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.954, "test_loss": 0.20908323516845703}, {"step": 5700, "epoch": 12, "train_loss": 0.12143538147211075, "cos_r_negg": [0.039261870086193085, 0.06593601405620575, 0.23642180860042572], "cos_innovation_negg": [0.3007679879665375, 0.34011101722717285, 0.652161717414856], "cos_apical_negg": [0.039261870086193085, 0.06593601405620575, 0.23642180860042572], "cos_Ac_negg": [0.29881545901298523, 0.3378750681877136, 0.6517305374145508], "r_norm": [3.444808006286621, 3.45407772064209, 3.4373631477355957]}, {"step": 5800, "epoch": 12, "train_loss": 0.07445135712623596, "cos_r_negg": [0.015866177156567574, 0.1466587483882904, 0.41649603843688965], "cos_innovation_negg": [0.36979877948760986, 0.43481582403182983, 0.7575439214706421], "cos_apical_negg": [0.015866177156567574, 0.1466587483882904, 0.41649603843688965], "cos_Ac_negg": [0.36949509382247925, 0.43381813168525696, 0.7574896812438965], "r_norm": [3.4374358654022217, 3.4438910484313965, 3.428027391433716]}, {"step": 5900, "epoch": 12, "train_loss": 0.09251949936151505, "cos_r_negg": [0.025119364261627197, 0.11391860991716385, 0.18940913677215576], "cos_innovation_negg": [0.28508108854293823, 0.33088159561157227, 0.6645840406417847], "cos_apical_negg": [0.025119364261627197, 0.11391860991716385, 0.18940913677215576], "cos_Ac_negg": [0.2831866443157196, 0.3292863965034485, 0.6646768450737], "r_norm": [3.457059621810913, 3.4668407440185547, 3.4477431774139404]}, {"step": 6000, "epoch": 12, "train_loss": 0.03523995727300644, "cos_r_negg": [0.04334006458520889, 0.03264812380075455, 0.32245323061943054], "cos_innovation_negg": [0.31931784749031067, 0.35719507932662964, 0.7935004830360413], "cos_apical_negg": [0.04334006458520889, 0.03264812380075455, 0.32245323061943054], "cos_Ac_negg": [0.31757938861846924, 0.3573112487792969, 0.793150007724762], "r_norm": [3.4484848976135254, 3.455357074737549, 3.444758415222168], "ldr": {"dL_sdil": -0.006154552102088928, "dL_bp": -0.013527214527130127, "ratio": 0.4549755671979167}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9574, "test_loss": 0.1899142219543457}, {"step": 6100, "epoch": 13, "train_loss": 0.0640493780374527, "cos_r_negg": [0.010680760256946087, 0.1182095855474472, 0.2559819519519806], "cos_innovation_negg": [0.31774404644966125, 0.3427150547504425, 0.7426396608352661], "cos_apical_negg": [0.010680760256946087, 0.1182095855474472, 0.2559819519519806], "cos_Ac_negg": [0.31749388575553894, 0.342637836933136, 0.7426520586013794], "r_norm": [3.4559106826782227, 3.4614415168762207, 3.4449143409729004]}, {"step": 6200, "epoch": 13, "train_loss": 0.07105721533298492, "cos_r_negg": [-0.0018275529146194458, 0.07961184531450272, 0.17910197377204895], "cos_innovation_negg": [0.27616631984710693, 0.381987601518631, 0.7015585899353027], "cos_apical_negg": [-0.0018275529146194458, 0.07961184531450272, 0.17910197377204895], "cos_Ac_negg": [0.2755950689315796, 0.3813946843147278, 0.7013348340988159], "r_norm": [3.438537836074829, 3.444610595703125, 3.433333396911621]}, {"step": 6300, "epoch": 13, "train_loss": 0.13348601758480072, "cos_r_negg": [0.026701023802161217, 0.09738105535507202, 0.3218535780906677], "cos_innovation_negg": [0.3533180356025696, 0.38160836696624756, 0.7684891819953918], "cos_apical_negg": [0.026701023802161217, 0.09738105535507202, 0.3218535780906677], "cos_Ac_negg": [0.3521589934825897, 0.381054550409317, 0.7686032652854919], "r_norm": [3.461559295654297, 3.467205762863159, 3.4523138999938965]}, {"step": 6400, "epoch": 13, "train_loss": 0.09620926529169083, "cos_r_negg": [0.019279517233371735, 0.11675151437520981, 0.26946234703063965], "cos_innovation_negg": [0.31829971075057983, 0.4088085889816284, 0.6620557308197021], "cos_apical_negg": [0.019279517233371735, 0.11675151437520981, 0.26946234703063965], "cos_Ac_negg": [0.3183930218219757, 0.4085819125175476, 0.6620597839355469], "r_norm": [3.446078300476074, 3.4481096267700195, 3.4378132820129395]}, {"step": 6500, "epoch": 13, "train_loss": 0.04643399268388748, "cos_r_negg": [0.021555813029408455, 0.10334792733192444, 0.34837692975997925], "cos_innovation_negg": [0.31326812505722046, 0.39714330434799194, 0.6596735119819641], "cos_apical_negg": [0.021555813029408455, 0.10334792733192444, 0.34837692975997925], "cos_Ac_negg": [0.31332555413246155, 0.39693838357925415, 0.6595549583435059], "r_norm": [3.4526519775390625, 3.4518423080444336, 3.441977024078369], "ldr": {"dL_sdil": -0.0101432204246521, "dL_bp": -0.018907442688941956, "ratio": 0.5364670723336041}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9584, "test_loss": 0.19562788200378417}, {"step": 6600, "epoch": 14, "train_loss": 0.09141496568918228, "cos_r_negg": [0.08231760561466217, 0.09222638607025146, 0.26672959327697754], "cos_innovation_negg": [0.29174041748046875, 0.36262786388397217, 0.6633678078651428], "cos_apical_negg": [0.08231760561466217, 0.09222638607025146, 0.26672959327697754], "cos_Ac_negg": [0.2914382517337799, 0.36211639642715454, 0.663729727268219], "r_norm": [3.442448616027832, 3.447157621383667, 3.432094097137451]}, {"step": 6700, "epoch": 14, "train_loss": 0.07480426132678986, "cos_r_negg": [0.06456546485424042, 0.08474172651767731, 0.19794611632823944], "cos_innovation_negg": [0.3047538697719574, 0.3929676115512848, 0.5812866687774658], "cos_apical_negg": [0.06456546485424042, 0.08474172651767731, 0.19794611632823944], "cos_Ac_negg": [0.30416980385780334, 0.39103347063064575, 0.5808819532394409], "r_norm": [3.43837308883667, 3.4449708461761475, 3.4330554008483887]}, {"step": 6800, "epoch": 14, "train_loss": 0.12511679530143738, "cos_r_negg": [0.03167286887764931, 0.1249806135892868, 0.28021514415740967], "cos_innovation_negg": [0.3121820092201233, 0.43177539110183716, 0.7645004987716675], "cos_apical_negg": [0.03167286887764931, 0.1249806135892868, 0.28021514415740967], "cos_Ac_negg": [0.31216001510620117, 0.4316609501838684, 0.7645552158355713], "r_norm": [3.439863920211792, 3.4437389373779297, 3.4313056468963623]}, {"step": 6900, "epoch": 14, "train_loss": 0.10097180306911469, "cos_r_negg": [0.02538044936954975, 0.0923779308795929, 0.1921561360359192], "cos_innovation_negg": [0.30251649022102356, 0.35801029205322266, 0.6664649248123169], "cos_apical_negg": [0.02538044936954975, 0.0923779308795929, 0.1921561360359192], "cos_Ac_negg": [0.3024083971977234, 0.3600451350212097, 0.6667554378509521], "r_norm": [3.441659450531006, 3.444916248321533, 3.4337821006774902]}, {"step": 7000, "epoch": 14, "train_loss": 0.164234459400177, "cos_r_negg": [-0.01903172954916954, 0.09667348116636276, 0.3766326308250427], "cos_innovation_negg": [0.31493452191352844, 0.4354863166809082, 0.6058841347694397], "cos_apical_negg": [-0.01903172954916954, 0.09667348116636276, 0.3766326308250427], "cos_Ac_negg": [0.31519296765327454, 0.4355885684490204, 0.6055018305778503], "r_norm": [3.4610681533813477, 3.464975595474243, 3.447291135787964], "ldr": {"dL_sdil": -0.0057841092348098755, "dL_bp": -0.00928407907485962, "ratio": 0.623013783938213}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9564, "test_loss": 0.19966592140197753}], "final": {"test_acc": 0.9564, "test_loss": 0.19966592140197753, "wall_s": 21.769463539123535, "cos_r_negg": [0.03210153803229332, 0.08044672012329102, 0.27450281381607056], "cos_innovation_negg": [0.25957590341567993, 0.32784104347229004, 0.7312723398208618], "cos_apical_negg": [0.03210153803229332, 0.08044672012329102, 0.27450281381607056], "cos_Ac_negg": [0.25565144419670105, 0.32587921619415283, 0.7313581705093384]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_raw_s4.json b/results/nuis_ctrl_v1_rho0p2_raw_s4.json new file mode 100644 index 0000000..05acf10 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_raw_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_raw_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.011956857517361641, 0.0204080268740654, 0.002555444138124585], "cos_innovation_negg": [-0.010422075167298317, 0.02403554506599903, 0.006981857120990753], "cos_apical_negg": [-0.011956857517361641, 0.0204080268740654, 0.002555444138124585], "cos_Ac_negg": [-0.010422931984066963, 0.024035122245550156, 0.006981394253671169], "r_norm": [5.565400123596191, 5.211086273193359, 5.183325290679932], "ldr": {"dL_sdil": 0.013530254364013672, "dL_bp": -0.2579684257507324, "ratio": -0.052449265155758144}}, {"step": 100, "epoch": 0, "train_loss": 0.4750537872314453, "cos_r_negg": [0.17289072275161743, 0.2665292024612427, 0.574877142906189], "cos_innovation_negg": [0.4410594701766968, 0.45542776584625244, 0.6422987580299377], "cos_apical_negg": [0.17289072275161743, 0.2665292024612427, 0.574877142906189], "cos_Ac_negg": [0.4392107129096985, 0.45346736907958984, 0.6418181657791138], "r_norm": [3.9450879096984863, 3.9541139602661133, 3.9377858638763428]}, {"step": 200, "epoch": 0, "train_loss": 0.42211389541625977, "cos_r_negg": [0.1335192322731018, 0.19010284543037415, 0.6028484106063843], "cos_innovation_negg": [0.4481812119483948, 0.4370300769805908, 0.6816461086273193], "cos_apical_negg": [0.1335192322731018, 0.19010284543037415, 0.6028484106063843], "cos_Ac_negg": [0.44762662053108215, 0.43662548065185547, 0.6815295219421387], "r_norm": [3.7121119499206543, 3.7272558212280273, 3.7217602729797363]}, {"step": 300, "epoch": 0, "train_loss": 0.4736926853656769, "cos_r_negg": [0.13582423329353333, 0.1760597825050354, 0.5747314691543579], "cos_innovation_negg": [0.4511204957962036, 0.4937560260295868, 0.7376217842102051], "cos_apical_negg": [0.13582423329353333, 0.1760597825050354, 0.5747314691543579], "cos_Ac_negg": [0.4509219527244568, 0.49370697140693665, 0.7374743223190308], "r_norm": [3.64347767829895, 3.666696071624756, 3.659895896911621]}, {"step": 400, "epoch": 0, "train_loss": 0.3019845187664032, "cos_r_negg": [0.15092137455940247, 0.2851549983024597, 0.622618556022644], "cos_innovation_negg": [0.49672600626945496, 0.5648658275604248, 0.8030319213867188], "cos_apical_negg": [0.15092137455940247, 0.2851549983024597, 0.622618556022644], "cos_Ac_negg": [0.4966853857040405, 0.5648783445358276, 0.8029477000236511], "r_norm": [3.649062395095825, 3.6652636528015137, 3.6506729125976562]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9203, "test_loss": 0.3187363037109375}, {"step": 500, "epoch": 1, "train_loss": 0.35978561639785767, "cos_r_negg": [0.11847931146621704, 0.2364443838596344, 0.4773114323616028], "cos_innovation_negg": [0.48284822702407837, 0.5253810882568359, 0.6845443248748779], "cos_apical_negg": [0.11847931146621704, 0.2364443838596344, 0.4773114323616028], "cos_Ac_negg": [0.4824906587600708, 0.5247441530227661, 0.6843276023864746], "r_norm": [3.6327996253967285, 3.647944450378418, 3.641751766204834], "ldr": {"dL_sdil": -0.011537283658981323, "dL_bp": -0.012406736612319946, "ratio": 0.9299208985805943}}, {"step": 600, "epoch": 1, "train_loss": 0.19335171580314636, "cos_r_negg": [0.13319805264472961, 0.23985205590724945, 0.5514774322509766], "cos_innovation_negg": [0.4706363081932068, 0.5092085003852844, 0.7829578518867493], "cos_apical_negg": [0.13319805264472961, 0.23985205590724945, 0.5514774322509766], "cos_Ac_negg": [0.47049611806869507, 0.5090044736862183, 0.7824071049690247], "r_norm": [3.580817222595215, 3.606771469116211, 3.6017746925354004]}, {"step": 700, "epoch": 1, "train_loss": 0.27263277769088745, "cos_r_negg": [0.12088846415281296, 0.2413065880537033, 0.67475825548172], "cos_innovation_negg": [0.48571228981018066, 0.5477913022041321, 0.8230524063110352], "cos_apical_negg": [0.12088846415281296, 0.2413065880537033, 0.67475825548172], "cos_Ac_negg": [0.48568645119667053, 0.5474509000778198, 0.8227138519287109], "r_norm": [3.570188283920288, 3.5988240242004395, 3.579251766204834]}, {"step": 800, "epoch": 1, "train_loss": 0.1933661699295044, "cos_r_negg": [0.09102176874876022, 0.2825019359588623, 0.6548992991447449], "cos_innovation_negg": [0.4500444531440735, 0.5296932458877563, 0.7932925224304199], "cos_apical_negg": [0.09102176874876022, 0.2825019359588623, 0.6548992991447449], "cos_Ac_negg": [0.449919730424881, 0.5296264290809631, 0.7931093573570251], "r_norm": [3.5590052604675293, 3.581848621368408, 3.5757079124450684]}, {"step": 900, "epoch": 1, "train_loss": 0.3662659227848053, "cos_r_negg": [0.08493069559335709, 0.209713414311409, 0.5847129225730896], "cos_innovation_negg": [0.4701869785785675, 0.5524070262908936, 0.8087437152862549], "cos_apical_negg": [0.08493069559335709, 0.209713414311409, 0.5847129225730896], "cos_Ac_negg": [0.4702374339103699, 0.5523287653923035, 0.8087341785430908], "r_norm": [3.5570943355560303, 3.563816547393799, 3.5509824752807617]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9387, "test_loss": 0.23808800735473634}, {"step": 1000, "epoch": 2, "train_loss": 0.12149813771247864, "cos_r_negg": [0.13127610087394714, 0.25622662901878357, 0.6465858817100525], "cos_innovation_negg": [0.4869597852230072, 0.5245695114135742, 0.8300520777702332], "cos_apical_negg": [0.13127610087394714, 0.25622662901878357, 0.6465858817100525], "cos_Ac_negg": [0.48692601919174194, 0.5246900916099548, 0.8300127983093262], "r_norm": [3.572998046875, 3.5996079444885254, 3.5870606899261475], "ldr": {"dL_sdil": -0.010308444499969482, "dL_bp": -0.011464014649391174, "ratio": 0.8992002204495559}}, {"step": 1100, "epoch": 2, "train_loss": 0.33518916368484497, "cos_r_negg": [0.08790959417819977, 0.2497953474521637, 0.5649596452713013], "cos_innovation_negg": [0.434362530708313, 0.5430561304092407, 0.7986395359039307], "cos_apical_negg": [0.08790959417819977, 0.2497953474521637, 0.5649596452713013], "cos_Ac_negg": [0.4341791272163391, 0.5426937937736511, 0.7985544204711914], "r_norm": [3.540452003479004, 3.5672266483306885, 3.5559024810791016]}, {"step": 1200, "epoch": 2, "train_loss": 0.37320321798324585, "cos_r_negg": [0.12525154650211334, 0.2557481527328491, 0.5923508405685425], "cos_innovation_negg": [0.4628221392631531, 0.5563222169876099, 0.8131669759750366], "cos_apical_negg": [0.12525154650211334, 0.2557481527328491, 0.5923508405685425], "cos_Ac_negg": [0.46274644136428833, 0.5564808249473572, 0.8130512833595276], "r_norm": [3.544528007507324, 3.546799659729004, 3.547952175140381]}, {"step": 1300, "epoch": 2, "train_loss": 0.2107192575931549, "cos_r_negg": [0.057930342853069305, 0.2530653476715088, 0.5829254388809204], "cos_innovation_negg": [0.450573205947876, 0.5354846715927124, 0.8413370847702026], "cos_apical_negg": [0.057930342853069305, 0.2530653476715088, 0.5829254388809204], "cos_Ac_negg": [0.45045751333236694, 0.5354177951812744, 0.8411997556686401], "r_norm": [3.5318074226379395, 3.535956859588623, 3.531754493713379]}, {"step": 1400, "epoch": 2, "train_loss": 0.26081418991088867, "cos_r_negg": [0.05684400349855423, 0.20282453298568726, 0.6184252500534058], "cos_innovation_negg": [0.4863371253013611, 0.5721842646598816, 0.8038393259048462], "cos_apical_negg": [0.05684400349855423, 0.20282453298568726, 0.6184252500534058], "cos_Ac_negg": [0.48632127046585083, 0.5720475912094116, 0.8036677837371826], "r_norm": [3.508273124694824, 3.528763771057129, 3.511690139770508]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9449, "test_loss": 0.21544832916259765}, {"step": 1500, "epoch": 3, "train_loss": 0.22312194108963013, "cos_r_negg": [0.05475308746099472, 0.18134787678718567, 0.5367257595062256], "cos_innovation_negg": [0.3670569062232971, 0.46349018812179565, 0.7474660277366638], "cos_apical_negg": [0.05475308746099472, 0.18134787678718567, 0.5367257595062256], "cos_Ac_negg": [0.3670521080493927, 0.4633214473724365, 0.7475631237030029], "r_norm": [3.5198721885681152, 3.535693645477295, 3.5233917236328125], "ldr": {"dL_sdil": -0.0068939924240112305, "dL_bp": -0.007081389427185059, "ratio": 0.9735366900661583}}, {"step": 1600, "epoch": 3, "train_loss": 0.21695445477962494, "cos_r_negg": [0.0818386971950531, 0.1643141508102417, 0.5312960147857666], "cos_innovation_negg": [0.43134045600891113, 0.5168430209159851, 0.7232449054718018], "cos_apical_negg": [0.0818386971950531, 0.1643141508102417, 0.5312960147857666], "cos_Ac_negg": [0.43129485845565796, 0.5169353485107422, 0.7230604290962219], "r_norm": [3.5030550956726074, 3.524172306060791, 3.5076117515563965]}, {"step": 1700, "epoch": 3, "train_loss": 0.14829111099243164, "cos_r_negg": [0.05950741469860077, 0.23633649945259094, 0.5927807092666626], "cos_innovation_negg": [0.47093746066093445, 0.5254899263381958, 0.7582101821899414], "cos_apical_negg": [0.05950741469860077, 0.23633649945259094, 0.5927807092666626], "cos_Ac_negg": [0.47104009985923767, 0.5257368087768555, 0.758141279220581], "r_norm": [3.506467819213867, 3.5251879692077637, 3.5080466270446777]}, {"step": 1800, "epoch": 3, "train_loss": 0.08678125590085983, "cos_r_negg": [0.055664125829935074, 0.19516195356845856, 0.5946141481399536], "cos_innovation_negg": [0.44033345580101013, 0.5145342350006104, 0.7555736899375916], "cos_apical_negg": [0.055664125829935074, 0.19516195356845856, 0.5946141481399536], "cos_Ac_negg": [0.4403555393218994, 0.5140544176101685, 0.7554497718811035], "r_norm": [3.530879020690918, 3.5565037727355957, 3.546597480773926]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9448, "test_loss": 0.23055702514648438}, {"step": 1900, "epoch": 4, "train_loss": 0.17653653025627136, "cos_r_negg": [0.09985731542110443, 0.249653622508049, 0.6090008020401001], "cos_innovation_negg": [0.4281187355518341, 0.5433008670806885, 0.726780354976654], "cos_apical_negg": [0.09985731542110443, 0.249653622508049, 0.6090008020401001], "cos_Ac_negg": [0.42824316024780273, 0.5437741875648499, 0.7255607843399048], "r_norm": [3.5450282096862793, 3.549558639526367, 3.5364203453063965]}, {"step": 2000, "epoch": 4, "train_loss": 0.21220366656780243, "cos_r_negg": [0.07960894703865051, 0.22611437737941742, 0.568345308303833], "cos_innovation_negg": [0.487714022397995, 0.5632803440093994, 0.8075501918792725], "cos_apical_negg": [0.07960894703865051, 0.22611437737941742, 0.568345308303833], "cos_Ac_negg": [0.4876750707626343, 0.5631519556045532, 0.8075096607208252], "r_norm": [3.4925031661987305, 3.5150599479675293, 3.4987144470214844], "ldr": {"dL_sdil": -0.009077027440071106, "dL_bp": -0.010187923908233643, "ratio": 0.8909594851543075}}, {"step": 2100, "epoch": 4, "train_loss": 0.08921875059604645, "cos_r_negg": [0.0939435362815857, 0.22298038005828857, 0.5617860555648804], "cos_innovation_negg": [0.4474589228630066, 0.5350453853607178, 0.8209345936775208], "cos_apical_negg": [0.0939435362815857, 0.22298038005828857, 0.5617860555648804], "cos_Ac_negg": [0.44751042127609253, 0.5347714424133301, 0.8209210634231567], "r_norm": [3.4979195594787598, 3.5182509422302246, 3.497817277908325]}, {"step": 2200, "epoch": 4, "train_loss": 0.11395369470119476, "cos_r_negg": [0.0822942778468132, 0.15268519520759583, 0.6701452732086182], "cos_innovation_negg": [0.3979695737361908, 0.4915057420730591, 0.8444207906723022], "cos_apical_negg": [0.0822942778468132, 0.15268519520759583, 0.6701452732086182], "cos_Ac_negg": [0.39813756942749023, 0.4918029010295868, 0.8443400859832764], "r_norm": [3.507755756378174, 3.515976905822754, 3.5087151527404785]}, {"step": 2300, "epoch": 4, "train_loss": 0.3364529609680176, "cos_r_negg": [0.0770307183265686, 0.21694979071617126, 0.6202293634414673], "cos_innovation_negg": [0.45409858226776123, 0.5880554914474487, 0.8336028456687927], "cos_apical_negg": [0.0770307183265686, 0.21694979071617126, 0.6202293634414673], "cos_Ac_negg": [0.4541519582271576, 0.5879000425338745, 0.8335167169570923], "r_norm": [3.4856796264648438, 3.501509666442871, 3.4909958839416504]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9479, "test_loss": 0.21531006774902345}, {"step": 2400, "epoch": 5, "train_loss": 0.22268988192081451, "cos_r_negg": [0.05968717485666275, 0.22637633979320526, 0.5694169402122498], "cos_innovation_negg": [0.43514344096183777, 0.5774223804473877, 0.7462446689605713], "cos_apical_negg": [0.05968717485666275, 0.22637633979320526, 0.5694169402122498], "cos_Ac_negg": [0.4350615441799164, 0.5765446424484253, 0.7457256317138672], "r_norm": [3.4932656288146973, 3.514101028442383, 3.500938892364502]}, {"step": 2500, "epoch": 5, "train_loss": 0.09341999888420105, "cos_r_negg": [0.019569875672459602, 0.19606637954711914, 0.4754149317741394], "cos_innovation_negg": [0.44084155559539795, 0.5889208316802979, 0.7859771251678467], "cos_apical_negg": [0.019569875672459602, 0.19606637954711914, 0.4754149317741394], "cos_Ac_negg": [0.44089609384536743, 0.5888198614120483, 0.7858589887619019], "r_norm": [3.501614570617676, 3.513394832611084, 3.5061142444610596], "ldr": {"dL_sdil": -0.009760826826095581, "dL_bp": -0.012443989515304565, "ratio": 0.7843808301261402}}, {"step": 2600, "epoch": 5, "train_loss": 0.17552168667316437, "cos_r_negg": [0.047727566212415695, 0.2561088502407074, 0.5877599120140076], "cos_innovation_negg": [0.39532262086868286, 0.5556920766830444, 0.7730623483657837], "cos_apical_negg": [0.047727566212415695, 0.2561088502407074, 0.5877599120140076], "cos_Ac_negg": [0.39529043436050415, 0.5555726289749146, 0.7726271152496338], "r_norm": [3.540367603302002, 3.5505189895629883, 3.543520927429199]}, {"step": 2700, "epoch": 5, "train_loss": 0.16440460085868835, "cos_r_negg": [0.08612929284572601, 0.22055748105049133, 0.6348462104797363], "cos_innovation_negg": [0.4664488434791565, 0.5508244037628174, 0.8962677717208862], "cos_apical_negg": [0.08612929284572601, 0.22055748105049133, 0.6348462104797363], "cos_Ac_negg": [0.4664746820926666, 0.550666332244873, 0.8961828351020813], "r_norm": [3.489314079284668, 3.5017409324645996, 3.4896774291992188]}, {"step": 2800, "epoch": 5, "train_loss": 0.29489392042160034, "cos_r_negg": [0.02368943765759468, 0.20780526101589203, 0.690775990486145], "cos_innovation_negg": [0.4487180709838867, 0.5598125457763672, 0.8652390241622925], "cos_apical_negg": [0.02368943765759468, 0.20780526101589203, 0.690775990486145], "cos_Ac_negg": [0.44904375076293945, 0.559325635433197, 0.8650967478752136], "r_norm": [3.4854655265808105, 3.5011043548583984, 3.488288402557373]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9493, "test_loss": 0.2088368148803711}, {"step": 2900, "epoch": 6, "train_loss": 0.08167441934347153, "cos_r_negg": [0.037279363721609116, 0.17545337975025177, 0.6865703463554382], "cos_innovation_negg": [0.41345784068107605, 0.5443288087844849, 0.8742119669914246], "cos_apical_negg": [0.037279363721609116, 0.17545337975025177, 0.6865703463554382], "cos_Ac_negg": [0.41342607140541077, 0.5439401865005493, 0.8740032911300659], "r_norm": [3.5052995681762695, 3.5192766189575195, 3.509443759918213]}, {"step": 3000, "epoch": 6, "train_loss": 0.42287296056747437, "cos_r_negg": [0.06268949806690216, 0.19135241210460663, 0.583855390548706], "cos_innovation_negg": [0.4322707951068878, 0.5566923022270203, 0.7725169062614441], "cos_apical_negg": [0.06268949806690216, 0.19135241210460663, 0.583855390548706], "cos_Ac_negg": [0.43265223503112793, 0.5564652681350708, 0.7723747491836548], "r_norm": [3.4727225303649902, 3.491758346557617, 3.4828524589538574], "ldr": {"dL_sdil": -0.007038027048110962, "dL_bp": -0.009097367525100708, "ratio": 0.773633364673046}}, {"step": 3100, "epoch": 6, "train_loss": 0.2642550468444824, "cos_r_negg": [0.01877940446138382, 0.2143135517835617, 0.557964563369751], "cos_innovation_negg": [0.4420211911201477, 0.5023664236068726, 0.8279370069503784], "cos_apical_negg": [0.01877940446138382, 0.2143135517835617, 0.557964563369751], "cos_Ac_negg": [0.44218194484710693, 0.5018340945243835, 0.8276281356811523], "r_norm": [3.5031251907348633, 3.5174241065979004, 3.5114688873291016]}, {"step": 3200, "epoch": 6, "train_loss": 0.21465060114860535, "cos_r_negg": [0.017408858984708786, 0.23596268892288208, 0.59035325050354], "cos_innovation_negg": [0.46481838822364807, 0.5952562093734741, 0.8424605131149292], "cos_apical_negg": [0.017408858984708786, 0.23596268892288208, 0.59035325050354], "cos_Ac_negg": [0.46495985984802246, 0.5914040207862854, 0.8423736095428467], "r_norm": [3.467249631881714, 3.4835939407348633, 3.471576690673828]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9511, "test_loss": 0.18839529342651368}, {"step": 3300, "epoch": 7, "train_loss": 0.13581418991088867, "cos_r_negg": [0.04182920604944229, 0.20473960041999817, 0.6690241098403931], "cos_innovation_negg": [0.4543980360031128, 0.5877878665924072, 0.850616455078125], "cos_apical_negg": [0.04182920604944229, 0.20473960041999817, 0.6690241098403931], "cos_Ac_negg": [0.45425844192504883, 0.5874686241149902, 0.8505607843399048], "r_norm": [3.4948437213897705, 3.496196985244751, 3.4916493892669678]}, {"step": 3400, "epoch": 7, "train_loss": 0.30225008726119995, "cos_r_negg": [0.04843023419380188, 0.18645831942558289, 0.6739591360092163], "cos_innovation_negg": [0.43378737568855286, 0.5751627087593079, 0.7943737506866455], "cos_apical_negg": [0.04843023419380188, 0.18645831942558289, 0.6739591360092163], "cos_Ac_negg": [0.43377113342285156, 0.5749982595443726, 0.7943326234817505], "r_norm": [3.4892115592956543, 3.4975619316101074, 3.4930148124694824]}, {"step": 3500, "epoch": 7, "train_loss": 0.13207051157951355, "cos_r_negg": [0.062053464353084564, 0.18894898891448975, 0.6230188012123108], "cos_innovation_negg": [0.426882803440094, 0.5420725345611572, 0.7607810497283936], "cos_apical_negg": [0.062053464353084564, 0.18894898891448975, 0.6230188012123108], "cos_Ac_negg": [0.4268535375595093, 0.5410705804824829, 0.7601210474967957], "r_norm": [3.4732272624969482, 3.4815917015075684, 3.4748454093933105], "ldr": {"dL_sdil": -0.005243748426437378, "dL_bp": -0.008037254214286804, "ratio": 0.6524303308854202}}, {"step": 3600, "epoch": 7, "train_loss": 0.05919833481311798, "cos_r_negg": [0.07355920970439911, 0.22196289896965027, 0.5801942348480225], "cos_innovation_negg": [0.3847639858722687, 0.5737966299057007, 0.8547283411026001], "cos_apical_negg": [0.07355920970439911, 0.22196289896965027, 0.5801942348480225], "cos_Ac_negg": [0.3846800923347473, 0.5735874176025391, 0.8545604944229126], "r_norm": [3.475571393966675, 3.490208387374878, 3.476595878601074]}, {"step": 3700, "epoch": 7, "train_loss": 0.13432708382606506, "cos_r_negg": [0.03509465605020523, 0.2035141885280609, 0.4782654941082001], "cos_innovation_negg": [0.4349420368671417, 0.5155513882637024, 0.8384146094322205], "cos_apical_negg": [0.03509465605020523, 0.2035141885280609, 0.4782654941082001], "cos_Ac_negg": [0.43494024872779846, 0.5154975056648254, 0.839094340801239], "r_norm": [3.4590110778808594, 3.4725236892700195, 3.4619650840759277]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9533, "test_loss": 0.19848698043823243}, {"step": 3800, "epoch": 8, "train_loss": 0.12582886219024658, "cos_r_negg": [0.047408945858478546, 0.2342541217803955, 0.5734213590621948], "cos_innovation_negg": [0.43895184993743896, 0.5944777727127075, 0.7722045183181763], "cos_apical_negg": [0.047408945858478546, 0.2342541217803955, 0.5734213590621948], "cos_Ac_negg": [0.4389934539794922, 0.5940360426902771, 0.7718954086303711], "r_norm": [3.464224338531494, 3.4798431396484375, 3.4673376083374023]}, {"step": 3900, "epoch": 8, "train_loss": 0.2349289357662201, "cos_r_negg": [0.05087464675307274, 0.20161005854606628, 0.448138564825058], "cos_innovation_negg": [0.4174814224243164, 0.5588624477386475, 0.8307009935379028], "cos_apical_negg": [0.05087464675307274, 0.20161005854606628, 0.448138564825058], "cos_Ac_negg": [0.4174521863460541, 0.5583441257476807, 0.8310527205467224], "r_norm": [3.4680304527282715, 3.4799129962921143, 3.469026565551758]}, {"step": 4000, "epoch": 8, "train_loss": 0.20901921391487122, "cos_r_negg": [0.04730428010225296, 0.26220422983169556, 0.5256604552268982], "cos_innovation_negg": [0.4598146378993988, 0.6259366273880005, 0.8224747180938721], "cos_apical_negg": [0.04730428010225296, 0.26220422983169556, 0.5256604552268982], "cos_Ac_negg": [0.45933812856674194, 0.6254764795303345, 0.8220374584197998], "r_norm": [3.4767117500305176, 3.491227865219116, 3.4723434448242188], "ldr": {"dL_sdil": -0.008212298154830933, "dL_bp": -0.010396763682365417, "ratio": 0.789889854740116}}, {"step": 4100, "epoch": 8, "train_loss": 0.08099746704101562, "cos_r_negg": [0.07924340665340424, 0.205892875790596, 0.425891250371933], "cos_innovation_negg": [0.40887951850891113, 0.5891261100769043, 0.755929172039032], "cos_apical_negg": [0.07924340665340424, 0.205892875790596, 0.425891250371933], "cos_Ac_negg": [0.4087597727775574, 0.589033842086792, 0.7560677528381348], "r_norm": [3.4688241481781006, 3.4768028259277344, 3.4709818363189697]}, {"step": 4200, "epoch": 8, "train_loss": 0.3419545888900757, "cos_r_negg": [0.03113587200641632, 0.1751873791217804, 0.3959196209907532], "cos_innovation_negg": [0.3896183967590332, 0.5375776290893555, 0.6484522223472595], "cos_apical_negg": [0.03113587200641632, 0.1751873791217804, 0.3959196209907532], "cos_Ac_negg": [0.3896007239818573, 0.537344217300415, 0.6576241254806519], "r_norm": [3.488478899002075, 3.492347478866577, 3.489715814590454]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9551, "test_loss": 0.1832194664001465}, {"step": 4300, "epoch": 9, "train_loss": 0.09275281429290771, "cos_r_negg": [0.053618162870407104, 0.19253221154212952, 0.5308492183685303], "cos_innovation_negg": [0.417873740196228, 0.5843667984008789, 0.7965271472930908], "cos_apical_negg": [0.053618162870407104, 0.19253221154212952, 0.5308492183685303], "cos_Ac_negg": [0.41739410161972046, 0.5841286182403564, 0.7967294454574585], "r_norm": [3.476356029510498, 3.486741065979004, 3.473093032836914]}, {"step": 4400, "epoch": 9, "train_loss": 0.07180020958185196, "cos_r_negg": [0.043264977633953094, 0.21635767817497253, 0.490176796913147], "cos_innovation_negg": [0.44119763374328613, 0.571502149105072, 0.759745717048645], "cos_apical_negg": [0.043264977633953094, 0.21635767817497253, 0.490176796913147], "cos_Ac_negg": [0.4413753151893616, 0.5715869665145874, 0.7595043182373047], "r_norm": [3.4643805027008057, 3.4721014499664307, 3.4618239402770996]}, {"step": 4500, "epoch": 9, "train_loss": 0.19398769736289978, "cos_r_negg": [0.03966209292411804, 0.21499177813529968, 0.5362972021102905], "cos_innovation_negg": [0.4309678077697754, 0.6087491512298584, 0.7102959156036377], "cos_apical_negg": [0.03966209292411804, 0.21499177813529968, 0.5362972021102905], "cos_Ac_negg": [0.43100830912590027, 0.6087126731872559, 0.7100602388381958], "r_norm": [3.486661195755005, 3.4934773445129395, 3.477720022201538], "ldr": {"dL_sdil": -0.006987050175666809, "dL_bp": -0.009711220860481262, "ratio": 0.7194821615168733}}, {"step": 4600, "epoch": 9, "train_loss": 0.0560777485370636, "cos_r_negg": [0.06531397998332977, 0.1800817847251892, 0.5799179077148438], "cos_innovation_negg": [0.45342251658439636, 0.5431491136550903, 0.8237646818161011], "cos_apical_negg": [0.06531397998332977, 0.1800817847251892, 0.5799179077148438], "cos_Ac_negg": [0.45336800813674927, 0.5423874258995056, 0.824088454246521], "r_norm": [3.4609766006469727, 3.4702136516571045, 3.459541082382202]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9542, "test_loss": 0.19892877578735352}, {"step": 4700, "epoch": 10, "train_loss": 0.17763805389404297, "cos_r_negg": [0.04531599208712578, 0.21647030115127563, 0.514319896697998], "cos_innovation_negg": [0.4301847815513611, 0.6016221642494202, 0.7524961829185486], "cos_apical_negg": [0.04531599208712578, 0.21647030115127563, 0.514319896697998], "cos_Ac_negg": [0.43001091480255127, 0.6015416383743286, 0.7521877884864807], "r_norm": [3.4750311374664307, 3.488917827606201, 3.480609655380249]}, {"step": 4800, "epoch": 10, "train_loss": 0.141770139336586, "cos_r_negg": [-0.003987464122474194, 0.23780937492847443, 0.4706122875213623], "cos_innovation_negg": [0.386099636554718, 0.6022370457649231, 0.7465347051620483], "cos_apical_negg": [-0.003987464122474194, 0.23780937492847443, 0.4706122875213623], "cos_Ac_negg": [0.38721463084220886, 0.601956307888031, 0.7459104061126709], "r_norm": [3.459686756134033, 3.472586154937744, 3.4658868312835693]}, {"step": 4900, "epoch": 10, "train_loss": 0.07928487658500671, "cos_r_negg": [0.026156213134527206, 0.1351097822189331, 0.3670060634613037], "cos_innovation_negg": [0.4061506390571594, 0.5064048767089844, 0.6853375434875488], "cos_apical_negg": [0.026156213134527206, 0.1351097822189331, 0.3670060634613037], "cos_Ac_negg": [0.4061405062675476, 0.5060154795646667, 0.6849254369735718], "r_norm": [3.463350534439087, 3.473461389541626, 3.4622201919555664]}, {"step": 5000, "epoch": 10, "train_loss": 0.027447305619716644, "cos_r_negg": [0.0516609251499176, 0.12610000371932983, 0.5079575181007385], "cos_innovation_negg": [0.399018257856369, 0.5668156147003174, 0.7164534330368042], "cos_apical_negg": [0.0516609251499176, 0.12610000371932983, 0.5079575181007385], "cos_Ac_negg": [0.399252325296402, 0.5665684938430786, 0.716035783290863], "r_norm": [3.4591786861419678, 3.472064733505249, 3.4581494331359863], "ldr": {"dL_sdil": -0.004805013537406921, "dL_bp": -0.007359206676483154, "ratio": 0.6529254780629642}}, {"step": 5100, "epoch": 10, "train_loss": 0.21249383687973022, "cos_r_negg": [0.031447723507881165, 0.17383599281311035, 0.43652480840682983], "cos_innovation_negg": [0.4170268177986145, 0.5315840244293213, 0.8075850009918213], "cos_apical_negg": [0.031447723507881165, 0.17383599281311035, 0.43652480840682983], "cos_Ac_negg": [0.4170093536376953, 0.5316397547721863, 0.8076450824737549], "r_norm": [3.4762325286865234, 3.4863297939300537, 3.477323055267334]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9559, "test_loss": 0.1883988006591797}, {"step": 5200, "epoch": 11, "train_loss": 0.08720330148935318, "cos_r_negg": [0.039067670702934265, 0.19694335758686066, 0.48266273736953735], "cos_innovation_negg": [0.43498164415359497, 0.5526241064071655, 0.8344366550445557], "cos_apical_negg": [0.039067670702934265, 0.19694335758686066, 0.48266273736953735], "cos_Ac_negg": [0.4349234700202942, 0.5524110794067383, 0.8342981934547424], "r_norm": [3.466118097305298, 3.475189447402954, 3.4649057388305664]}, {"step": 5300, "epoch": 11, "train_loss": 0.1388130933046341, "cos_r_negg": [0.031038496643304825, 0.14290937781333923, 0.5434865951538086], "cos_innovation_negg": [0.4273046851158142, 0.507470965385437, 0.8129115104675293], "cos_apical_negg": [0.031038496643304825, 0.14290937781333923, 0.5434865951538086], "cos_Ac_negg": [0.4272947609424591, 0.507317066192627, 0.8124746680259705], "r_norm": [3.468296527862549, 3.478717088699341, 3.466716766357422]}, {"step": 5400, "epoch": 11, "train_loss": 0.27817630767822266, "cos_r_negg": [0.017896460369229317, 0.1772950440645218, 0.48078250885009766], "cos_innovation_negg": [0.3642120063304901, 0.5362515449523926, 0.704262375831604], "cos_apical_negg": [0.017896460369229317, 0.1772950440645218, 0.48078250885009766], "cos_Ac_negg": [0.3674371838569641, 0.53584885597229, 0.7039588689804077], "r_norm": [3.456787586212158, 3.4661812782287598, 3.449216365814209]}, {"step": 5500, "epoch": 11, "train_loss": 0.20167073607444763, "cos_r_negg": [0.025979511439800262, 0.24400228261947632, 0.4427090287208557], "cos_innovation_negg": [0.3989015817642212, 0.5896512269973755, 0.702342689037323], "cos_apical_negg": [0.025979511439800262, 0.24400228261947632, 0.4427090287208557], "cos_Ac_negg": [0.3991009593009949, 0.5893043279647827, 0.7025435566902161], "r_norm": [3.4555277824401855, 3.4655354022979736, 3.454407215118408], "ldr": {"dL_sdil": -0.004147320985794067, "dL_bp": -0.006773233413696289, "ratio": 0.61231035939315}}, {"step": 5600, "epoch": 11, "train_loss": 0.11804436147212982, "cos_r_negg": [0.049389004707336426, 0.17493867874145508, 0.38279426097869873], "cos_innovation_negg": [0.4189903140068054, 0.5583648681640625, 0.8230645060539246], "cos_apical_negg": [0.049389004707336426, 0.17493867874145508, 0.38279426097869873], "cos_Ac_negg": [0.41872096061706543, 0.5583170652389526, 0.822784423828125], "r_norm": [3.485067367553711, 3.497248888015747, 3.486901044845581]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9546, "test_loss": 0.197890909576416}, {"step": 5700, "epoch": 12, "train_loss": 0.07822814583778381, "cos_r_negg": [0.042551275342702866, 0.2011890709400177, 0.44901496171951294], "cos_innovation_negg": [0.41427987813949585, 0.5747051239013672, 0.7529419660568237], "cos_apical_negg": [0.042551275342702866, 0.2011890709400177, 0.44901496171951294], "cos_Ac_negg": [0.40688616037368774, 0.573483943939209, 0.7563733458518982], "r_norm": [3.4518208503723145, 3.4615461826324463, 3.4539847373962402]}, {"step": 5800, "epoch": 12, "train_loss": 0.043576110154390335, "cos_r_negg": [0.04847339540719986, 0.15626055002212524, 0.4628142714500427], "cos_innovation_negg": [0.4235988259315491, 0.5542389154434204, 0.7477295398712158], "cos_apical_negg": [0.04847339540719986, 0.15626055002212524, 0.4628142714500427], "cos_Ac_negg": [0.4233926236629486, 0.5539628267288208, 0.7543405294418335], "r_norm": [3.4550275802612305, 3.4634969234466553, 3.453120231628418]}, {"step": 5900, "epoch": 12, "train_loss": 0.1297622174024582, "cos_r_negg": [0.018939029425382614, 0.1774609386920929, 0.49215948581695557], "cos_innovation_negg": [0.4190130829811096, 0.5573451519012451, 0.769503116607666], "cos_apical_negg": [0.018939029425382614, 0.1774609386920929, 0.49215948581695557], "cos_Ac_negg": [0.41877099871635437, 0.557030200958252, 0.7704774141311646], "r_norm": [3.4489479064941406, 3.457632541656494, 3.4471538066864014]}, {"step": 6000, "epoch": 12, "train_loss": 0.06887461245059967, "cos_r_negg": [0.022489607334136963, 0.18150678277015686, 0.5400077104568481], "cos_innovation_negg": [0.39491206407546997, 0.5437368154525757, 0.8377511501312256], "cos_apical_negg": [0.022489607334136963, 0.18150678277015686, 0.5400077104568481], "cos_Ac_negg": [0.39466768503189087, 0.5434008240699768, 0.8379849195480347], "r_norm": [3.4569315910339355, 3.4669337272644043, 3.4562478065490723], "ldr": {"dL_sdil": -0.004834681749343872, "dL_bp": -0.007802397012710571, "ratio": 0.6196405721815855}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9541, "test_loss": 0.19915148162841798}, {"step": 6100, "epoch": 13, "train_loss": 0.12441891431808472, "cos_r_negg": [0.037758029997348785, 0.21973922848701477, 0.5656598210334778], "cos_innovation_negg": [0.4033668041229248, 0.5921123027801514, 0.80461186170578], "cos_apical_negg": [0.037758029997348785, 0.21973922848701477, 0.5656598210334778], "cos_Ac_negg": [0.40334028005599976, 0.5919814705848694, 0.804319441318512], "r_norm": [3.448150634765625, 3.4538209438323975, 3.447100877761841]}, {"step": 6200, "epoch": 13, "train_loss": 0.03186078742146492, "cos_r_negg": [0.024448025971651077, 0.18667493760585785, 0.4622846245765686], "cos_innovation_negg": [0.35262954235076904, 0.5385372638702393, 0.7032085657119751], "cos_apical_negg": [0.024448025971651077, 0.18667493760585785, 0.4622846245765686], "cos_Ac_negg": [0.3525902032852173, 0.5382406115531921, 0.7020827531814575], "r_norm": [3.4546518325805664, 3.460237979888916, 3.4550435543060303]}, {"step": 6300, "epoch": 13, "train_loss": 0.03825074061751366, "cos_r_negg": [0.020515820011496544, 0.21622180938720703, 0.44693082571029663], "cos_innovation_negg": [0.4155089259147644, 0.5973267555236816, 0.6766473054885864], "cos_apical_negg": [0.020515820011496544, 0.21622180938720703, 0.44693082571029663], "cos_Ac_negg": [0.41552019119262695, 0.5969445705413818, 0.6782469749450684], "r_norm": [3.448761463165283, 3.455890655517578, 3.4471638202667236]}, {"step": 6400, "epoch": 13, "train_loss": 0.03932960703969002, "cos_r_negg": [0.026593996211886406, 0.2256653904914856, 0.39923524856567383], "cos_innovation_negg": [0.41374319791793823, 0.5692796111106873, 0.80968177318573], "cos_apical_negg": [0.026593996211886406, 0.2256653904914856, 0.39923524856567383], "cos_Ac_negg": [0.4138638973236084, 0.5679604411125183, 0.8093268871307373], "r_norm": [3.4697105884552, 3.482572555541992, 3.4712765216827393]}, {"step": 6500, "epoch": 13, "train_loss": 0.011997969821095467, "cos_r_negg": [0.041799116879701614, 0.12720957398414612, 0.3751963973045349], "cos_innovation_negg": [0.40569421648979187, 0.5459054708480835, 0.8217697143554688], "cos_apical_negg": [0.041799116879701614, 0.12720957398414612, 0.3751963973045349], "cos_Ac_negg": [0.4057433307170868, 0.5467766523361206, 0.8217895030975342], "r_norm": [3.4552578926086426, 3.4635722637176514, 3.454599380493164], "ldr": {"dL_sdil": -0.004309207201004028, "dL_bp": -0.00781385600566864, "ratio": 0.551482801561466}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9544, "test_loss": 0.1913643699645996}, {"step": 6600, "epoch": 14, "train_loss": 0.02708890289068222, "cos_r_negg": [9.342096745967865e-05, 0.13607776165008545, 0.5366811752319336], "cos_innovation_negg": [0.36983031034469604, 0.5505750179290771, 0.818997859954834], "cos_apical_negg": [9.342096745967865e-05, 0.13607776165008545, 0.5366811752319336], "cos_Ac_negg": [0.370279997587204, 0.5495230555534363, 0.8190932273864746], "r_norm": [3.4489996433258057, 3.452692747116089, 3.4436049461364746]}, {"step": 6700, "epoch": 14, "train_loss": 0.08523418009281158, "cos_r_negg": [0.02637433633208275, 0.1678479015827179, 0.5084254741668701], "cos_innovation_negg": [0.4090483784675598, 0.534423828125, 0.7576724290847778], "cos_apical_negg": [0.02637433633208275, 0.1678479015827179, 0.5084254741668701], "cos_Ac_negg": [0.4092547595500946, 0.5366582274436951, 0.7602380514144897], "r_norm": [3.457058906555176, 3.466118097305298, 3.4575705528259277]}, {"step": 6800, "epoch": 14, "train_loss": 0.09282908588647842, "cos_r_negg": [0.020635008811950684, 0.18371716141700745, 0.39045971632003784], "cos_innovation_negg": [0.40185490250587463, 0.5615551471710205, 0.7576956152915955], "cos_apical_negg": [0.020635008811950684, 0.18371716141700745, 0.39045971632003784], "cos_Ac_negg": [0.4020015597343445, 0.5608091354370117, 0.7574525475502014], "r_norm": [3.459503650665283, 3.4631452560424805, 3.455723285675049]}, {"step": 6900, "epoch": 14, "train_loss": 0.07403425872325897, "cos_r_negg": [0.004958726465702057, 0.21029961109161377, 0.4955659508705139], "cos_innovation_negg": [0.4068952202796936, 0.6397120356559753, 0.7893516421318054], "cos_apical_negg": [0.004958726465702057, 0.21029961109161377, 0.4955659508705139], "cos_Ac_negg": [0.40709972381591797, 0.6391990780830383, 0.7891943454742432], "r_norm": [3.4468507766723633, 3.451991319656372, 3.4429473876953125]}, {"step": 7000, "epoch": 14, "train_loss": 0.1459781378507614, "cos_r_negg": [0.006495658308267593, 0.20425459742546082, 0.4501078724861145], "cos_innovation_negg": [0.42327141761779785, 0.6309816837310791, 0.8001081943511963], "cos_apical_negg": [0.006495658308267593, 0.20425459742546082, 0.4501078724861145], "cos_Ac_negg": [0.4234600067138672, 0.6317270994186401, 0.8001537919044495], "r_norm": [3.457538604736328, 3.4639830589294434, 3.45184326171875], "ldr": {"dL_sdil": -0.003236740827560425, "dL_bp": -0.0056077539920806885, "ratio": 0.577190231977254}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9532, "test_loss": 0.2099518783569336}], "final": {"test_acc": 0.9532, "test_loss": 0.2099518783569336, "wall_s": 21.86707639694214, "cos_r_negg": [-0.013040256686508656, 0.19956542551517487, 0.3413127064704895], "cos_innovation_negg": [0.35866662859916687, 0.5476642847061157, 0.7576143145561218], "cos_apical_negg": [-0.013040256686508656, 0.19956542551517487, 0.3413127064704895], "cos_Ac_negg": [0.3587781488895416, 0.5472832918167114, 0.760552167892456]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_residual_s0.json b/results/nuis_ctrl_v1_rho0p2_residual_s0.json new file mode 100644 index 0000000..f6feca7 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_residual_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_residual_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.019523506984114647, -0.025533119216561317, 0.033746011555194855], "cos_innovation_negg": [-0.019523506984114647, -0.025533119216561317, 0.033746011555194855], "cos_apical_negg": [-0.01901932805776596, -0.025649813935160637, 0.030604533851146698], "cos_Ac_negg": [-0.019523320719599724, -0.02553311362862587, 0.03374597430229187], "r_norm": [4.898260116577148, 4.826771259307861, 4.823907852172852], "ldr": {"dL_sdil": -0.009198188781738281, "dL_bp": -0.24068951606750488, "ratio": 0.03821599266981997}}, {"step": 100, "epoch": 0, "train_loss": 0.20606768131256104, "cos_r_negg": [0.4568517208099365, 0.3804790675640106, 0.3602581322193146], "cos_innovation_negg": [0.4568517208099365, 0.3804790675640106, 0.3602581322193146], "cos_apical_negg": [0.11668631434440613, 0.11449643224477768, 0.16664618253707886], "cos_Ac_negg": [0.4580104351043701, 0.3817555010318756, 0.361117422580719], "r_norm": [0.8251547813415527, 0.8088545799255371, 0.8064684867858887]}, {"step": 200, "epoch": 0, "train_loss": 0.33056187629699707, "cos_r_negg": [0.4819785952568054, 0.4062795042991638, 0.3873841166496277], "cos_innovation_negg": [0.4819785952568054, 0.4062795042991638, 0.3873841166496277], "cos_apical_negg": [0.10751524567604065, 0.10858698189258575, 0.1609276384115219], "cos_Ac_negg": [0.4838070273399353, 0.4091993570327759, 0.390473872423172], "r_norm": [0.74756920337677, 0.730886697769165, 0.7294856309890747]}, {"step": 300, "epoch": 0, "train_loss": 0.15221649408340454, "cos_r_negg": [0.49626708030700684, 0.4471138119697571, 0.42699939012527466], "cos_innovation_negg": [0.49626708030700684, 0.4471138119697571, 0.42699939012527466], "cos_apical_negg": [0.09722556173801422, 0.09840162098407745, 0.13177180290222168], "cos_Ac_negg": [0.49734193086624146, 0.44760507345199585, 0.4274216890335083], "r_norm": [0.5708848237991333, 0.56050044298172, 0.5594212412834167]}, {"step": 400, "epoch": 0, "train_loss": 0.14678801596164703, "cos_r_negg": [0.507038950920105, 0.475298136472702, 0.44862401485443115], "cos_innovation_negg": [0.507038950920105, 0.475298136472702, 0.44862401485443115], "cos_apical_negg": [0.09293738007545471, 0.10707753896713257, 0.13955432176589966], "cos_Ac_negg": [0.5077444314956665, 0.4754383862018585, 0.4486578106880188], "r_norm": [0.5688608288764954, 0.5588194727897644, 0.5559911131858826]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9448, "test_loss": 0.18991070022583006}, {"step": 500, "epoch": 1, "train_loss": 0.200868621468544, "cos_r_negg": [0.5162969827651978, 0.48728224635124207, 0.48449674248695374], "cos_innovation_negg": [0.5162969827651978, 0.48728224635124207, 0.48449674248695374], "cos_apical_negg": [0.08657191693782806, 0.10910124331712723, 0.15178021788597107], "cos_Ac_negg": [0.5193557739257812, 0.4889791011810303, 0.48533231019973755], "r_norm": [0.5458161234855652, 0.5269564986228943, 0.5282394886016846], "ldr": {"dL_sdil": -0.013976424932479858, "dL_bp": -0.011787191033363342, "ratio": 1.1857299073986283}}, {"step": 600, "epoch": 1, "train_loss": 0.06455937772989273, "cos_r_negg": [0.5252150893211365, 0.49741560220718384, 0.47792404890060425], "cos_innovation_negg": [0.5252150893211365, 0.49741560220718384, 0.47792404890060425], "cos_apical_negg": [0.07249660044908524, 0.09262460470199585, 0.1474091112613678], "cos_Ac_negg": [0.5307551622390747, 0.49993205070495605, 0.4801490306854248], "r_norm": [0.4578191637992859, 0.44566091895103455, 0.4451056718826294]}, {"step": 700, "epoch": 1, "train_loss": 0.18288546800613403, "cos_r_negg": [0.5178865790367126, 0.5105553865432739, 0.4914657473564148], "cos_innovation_negg": [0.5178865790367126, 0.5105553865432739, 0.4914657473564148], "cos_apical_negg": [0.06589733809232712, 0.0876186192035675, 0.15391361713409424], "cos_Ac_negg": [0.5210632085800171, 0.5110137462615967, 0.49143391847610474], "r_norm": [0.4621240496635437, 0.4475422501564026, 0.44855743646621704]}, {"step": 800, "epoch": 1, "train_loss": 0.1488797962665558, "cos_r_negg": [0.5349124073982239, 0.5300829410552979, 0.5068248510360718], "cos_innovation_negg": [0.5349124073982239, 0.5300829410552979, 0.5068248510360718], "cos_apical_negg": [0.0719074159860611, 0.08885012567043304, 0.15127798914909363], "cos_Ac_negg": [0.5383347868919373, 0.5316882729530334, 0.5075443983078003], "r_norm": [0.3949119448661804, 0.38480696082115173, 0.38360413908958435]}, {"step": 900, "epoch": 1, "train_loss": 0.17730845510959625, "cos_r_negg": [0.554450511932373, 0.5506994724273682, 0.5195983648300171], "cos_innovation_negg": [0.554450511932373, 0.5506994724273682, 0.5195983648300171], "cos_apical_negg": [0.053759295493364334, 0.08073358237743378, 0.14462418854236603], "cos_Ac_negg": [0.5558062791824341, 0.5511731505393982, 0.51972496509552], "r_norm": [0.45816269516944885, 0.44765082001686096, 0.44630134105682373]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9543, "test_loss": 0.15480947380065918}, {"step": 1000, "epoch": 2, "train_loss": 0.03710019960999489, "cos_r_negg": [0.5558743476867676, 0.5464800000190735, 0.5298841595649719], "cos_innovation_negg": [0.5558743476867676, 0.5464800000190735, 0.5298841595649719], "cos_apical_negg": [0.05135207623243332, 0.06875786185264587, 0.13144917786121368], "cos_Ac_negg": [0.5585004091262817, 0.547971785068512, 0.5297491550445557], "r_norm": [0.3376040458679199, 0.32865121960639954, 0.3271465599536896], "ldr": {"dL_sdil": -0.008514806628227234, "dL_bp": -0.007336333394050598, "ratio": 1.1606351798477859}}, {"step": 1100, "epoch": 2, "train_loss": 0.07798951119184494, "cos_r_negg": [0.5498685836791992, 0.5463867783546448, 0.5243546366691589], "cos_innovation_negg": [0.5498685836791992, 0.5463867783546448, 0.5243546366691589], "cos_apical_negg": [0.07133688032627106, 0.07786333560943604, 0.13750848174095154], "cos_Ac_negg": [0.5545761585235596, 0.5487866401672363, 0.5241233110427856], "r_norm": [0.37861159443855286, 0.3637913465499878, 0.36582499742507935]}, {"step": 1200, "epoch": 2, "train_loss": 0.14072027802467346, "cos_r_negg": [0.5525054931640625, 0.5577645301818848, 0.5312122106552124], "cos_innovation_negg": [0.5525054931640625, 0.5577645301818848, 0.5312122106552124], "cos_apical_negg": [0.043642252683639526, 0.068376824259758, 0.128059983253479], "cos_Ac_negg": [0.554104745388031, 0.5589102506637573, 0.5304606556892395], "r_norm": [0.31428849697113037, 0.3077391982078552, 0.3076714873313904]}, {"step": 1300, "epoch": 2, "train_loss": 0.11660172045230865, "cos_r_negg": [0.5418621897697449, 0.5479443073272705, 0.5228186845779419], "cos_innovation_negg": [0.5418621897697449, 0.5479443073272705, 0.5228186845779419], "cos_apical_negg": [0.03705895319581032, 0.07983006536960602, 0.1325947642326355], "cos_Ac_negg": [0.5479830503463745, 0.5507307052612305, 0.5222512483596802], "r_norm": [0.2982795834541321, 0.291095495223999, 0.2893028259277344]}, {"step": 1400, "epoch": 2, "train_loss": 0.1453935205936432, "cos_r_negg": [0.5396968126296997, 0.544838011264801, 0.5175241231918335], "cos_innovation_negg": [0.5396968126296997, 0.544838011264801, 0.5175241231918335], "cos_apical_negg": [0.04281812161207199, 0.07184118032455444, 0.13447293639183044], "cos_Ac_negg": [0.55027836561203, 0.551777720451355, 0.5188418626785278], "r_norm": [0.3004603683948517, 0.2916700839996338, 0.29145216941833496]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.965, "test_loss": 0.11640464172363281}, {"step": 1500, "epoch": 3, "train_loss": 0.05794712156057358, "cos_r_negg": [0.5365474820137024, 0.5483611226081848, 0.5290035605430603], "cos_innovation_negg": [0.5365474820137024, 0.5483611226081848, 0.5290035605430603], "cos_apical_negg": [0.03903874382376671, 0.0680883601307869, 0.11794716864824295], "cos_Ac_negg": [0.553656280040741, 0.5600674152374268, 0.5332503318786621], "r_norm": [0.2801234722137451, 0.2716987431049347, 0.2720698118209839], "ldr": {"dL_sdil": -0.006641030311584473, "dL_bp": -0.0062914565205574036, "ratio": 1.0555632531012227}}, {"step": 1600, "epoch": 3, "train_loss": 0.04811064898967743, "cos_r_negg": [0.531104564666748, 0.5472593307495117, 0.5583807826042175], "cos_innovation_negg": [0.531104564666748, 0.5472593307495117, 0.5583807826042175], "cos_apical_negg": [0.04239968955516815, 0.07670260965824127, 0.12730857729911804], "cos_Ac_negg": [0.5430436134338379, 0.5563105344772339, 0.5606163740158081], "r_norm": [0.2991618514060974, 0.28990256786346436, 0.29055726528167725]}, {"step": 1700, "epoch": 3, "train_loss": 0.08352155983448029, "cos_r_negg": [0.5660141706466675, 0.5690721869468689, 0.559603214263916], "cos_innovation_negg": [0.5660141706466675, 0.5690721869468689, 0.559603214263916], "cos_apical_negg": [0.03792407363653183, 0.07880504429340363, 0.12992222607135773], "cos_Ac_negg": [0.5771263837814331, 0.5787938833236694, 0.5633704662322998], "r_norm": [0.30538299679756165, 0.29572775959968567, 0.2982964515686035]}, {"step": 1800, "epoch": 3, "train_loss": 0.030646156519651413, "cos_r_negg": [0.5485644340515137, 0.5643365383148193, 0.5444592833518982], "cos_innovation_negg": [0.5485644340515137, 0.5643365383148193, 0.5444592833518982], "cos_apical_negg": [0.03874468058347702, 0.06933817267417908, 0.12000374495983124], "cos_Ac_negg": [0.563870370388031, 0.5779374837875366, 0.549401044845581], "r_norm": [0.260032057762146, 0.25294506549835205, 0.2516686022281647]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9653, "test_loss": 0.11902983360290527}, {"step": 1900, "epoch": 4, "train_loss": 0.03175019472837448, "cos_r_negg": [0.5577069520950317, 0.5627659559249878, 0.565239667892456], "cos_innovation_negg": [0.5577069520950317, 0.5627659559249878, 0.565239667892456], "cos_apical_negg": [0.03871120512485504, 0.0746726244688034, 0.13173167407512665], "cos_Ac_negg": [0.5703960657119751, 0.5756616592407227, 0.570755124092102], "r_norm": [0.29211440682411194, 0.2857474684715271, 0.28569066524505615]}, {"step": 2000, "epoch": 4, "train_loss": 0.08024226129055023, "cos_r_negg": [0.5266743898391724, 0.553314208984375, 0.5544223785400391], "cos_innovation_negg": [0.5266743898391724, 0.553314208984375, 0.5544223785400391], "cos_apical_negg": [0.03658125549554825, 0.06792008876800537, 0.11263023316860199], "cos_Ac_negg": [0.5573714375495911, 0.5773229598999023, 0.5672880411148071], "r_norm": [0.21113789081573486, 0.2038601040840149, 0.20290419459342957], "ldr": {"dL_sdil": -0.004576161503791809, "dL_bp": -0.004620671272277832, "ratio": 0.9903672505869302}}, {"step": 2100, "epoch": 4, "train_loss": 0.02029000222682953, "cos_r_negg": [0.5275942087173462, 0.5527107119560242, 0.5520141124725342], "cos_innovation_negg": [0.5275942087173462, 0.5527107119560242, 0.5520141124725342], "cos_apical_negg": [0.040275901556015015, 0.07150642573833466, 0.12067275494337082], "cos_Ac_negg": [0.550916314125061, 0.5742125511169434, 0.5672310590744019], "r_norm": [0.2565217614173889, 0.24834167957305908, 0.24823063611984253]}, {"step": 2200, "epoch": 4, "train_loss": 0.00951699260622263, "cos_r_negg": [0.5344455242156982, 0.5552586913108826, 0.5587232112884521], "cos_innovation_negg": [0.5344455242156982, 0.5552586913108826, 0.5587232112884521], "cos_apical_negg": [0.035655803978443146, 0.06762439757585526, 0.12687289714813232], "cos_Ac_negg": [0.5651898384094238, 0.5814754962921143, 0.574934720993042], "r_norm": [0.2500680088996887, 0.24069690704345703, 0.2411661297082901]}, {"step": 2300, "epoch": 4, "train_loss": 0.1815711259841919, "cos_r_negg": [0.4886285364627838, 0.5215216279029846, 0.5413792133331299], "cos_innovation_negg": [0.4886285364627838, 0.5215216279029846, 0.5413792133331299], "cos_apical_negg": [0.027654336765408516, 0.0623752623796463, 0.12552689015865326], "cos_Ac_negg": [0.5431786179542542, 0.573932945728302, 0.577804684638977], "r_norm": [0.25167208909988403, 0.24495288729667664, 0.2447163462638855]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9691, "test_loss": 0.11393807468414306}, {"step": 2400, "epoch": 5, "train_loss": 0.10122163593769073, "cos_r_negg": [0.5314934253692627, 0.5587554574012756, 0.5576316118240356], "cos_innovation_negg": [0.5314934253692627, 0.5587554574012756, 0.5576316118240356], "cos_apical_negg": [0.02344726212322712, 0.0665503516793251, 0.11944691836833954], "cos_Ac_negg": [0.5785295963287354, 0.6011965274810791, 0.5858073234558105], "r_norm": [0.22358477115631104, 0.218226358294487, 0.21524173021316528]}, {"step": 2500, "epoch": 5, "train_loss": 0.038397856056690216, "cos_r_negg": [0.4825775623321533, 0.5174044966697693, 0.5375474691390991], "cos_innovation_negg": [0.4825775623321533, 0.5174044966697693, 0.5375474691390991], "cos_apical_negg": [0.02063409797847271, 0.055779747664928436, 0.11611965298652649], "cos_Ac_negg": [0.5568516254425049, 0.5858498811721802, 0.5866438150405884], "r_norm": [0.19941456615924835, 0.19570526480674744, 0.19538924098014832], "ldr": {"dL_sdil": -0.005884647369384766, "dL_bp": -0.006330087780952454, "ratio": 0.9296312425701204}}, {"step": 2600, "epoch": 5, "train_loss": 0.05920883268117905, "cos_r_negg": [0.4925805628299713, 0.523348331451416, 0.5389696359634399], "cos_innovation_negg": [0.4925805628299713, 0.523348331451416, 0.5389696359634399], "cos_apical_negg": [0.025194434449076653, 0.05186089873313904, 0.11524221301078796], "cos_Ac_negg": [0.5781121253967285, 0.600778341293335, 0.5976366996765137], "r_norm": [0.18396976590156555, 0.1793268918991089, 0.17820721864700317]}, {"step": 2700, "epoch": 5, "train_loss": 0.016289614140987396, "cos_r_negg": [0.5190460681915283, 0.5432322025299072, 0.5588918328285217], "cos_innovation_negg": [0.5190460681915283, 0.5432322025299072, 0.5588918328285217], "cos_apical_negg": [0.01423963438719511, 0.05856696516275406, 0.129440039396286], "cos_Ac_negg": [0.5946996212005615, 0.6120826005935669, 0.6044851541519165], "r_norm": [0.18712875247001648, 0.1825256049633026, 0.18095046281814575]}, {"step": 2800, "epoch": 5, "train_loss": 0.10968293249607086, "cos_r_negg": [0.49568527936935425, 0.52587890625, 0.5606033205986023], "cos_innovation_negg": [0.49568527936935425, 0.52587890625, 0.5606033205986023], "cos_apical_negg": [0.01931498944759369, 0.06305486708879471, 0.14198926091194153], "cos_Ac_negg": [0.5769494771957397, 0.6030417680740356, 0.6160200834274292], "r_norm": [0.22943010926246643, 0.22177505493164062, 0.22182336449623108]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9709, "test_loss": 0.11041314105987549}, {"step": 2900, "epoch": 6, "train_loss": 0.008864540606737137, "cos_r_negg": [0.46098676323890686, 0.5032291412353516, 0.5465518236160278], "cos_innovation_negg": [0.46098676323890686, 0.5032291412353516, 0.5465518236160278], "cos_apical_negg": [0.016328910365700722, 0.060745444148778915, 0.13262057304382324], "cos_Ac_negg": [0.5510411262512207, 0.5917937755584717, 0.6131106615066528], "r_norm": [0.18359653651714325, 0.17735886573791504, 0.17770491540431976]}, {"step": 3000, "epoch": 6, "train_loss": 0.08613086491823196, "cos_r_negg": [0.4443424642086029, 0.49303972721099854, 0.526557207107544], "cos_innovation_negg": [0.4443424642086029, 0.49303972721099854, 0.526557207107544], "cos_apical_negg": [0.017452478408813477, 0.05924266576766968, 0.13208407163619995], "cos_Ac_negg": [0.5434337258338928, 0.5934004783630371, 0.6081597805023193], "r_norm": [0.18291375041007996, 0.17817261815071106, 0.17709821462631226], "ldr": {"dL_sdil": -0.004899367690086365, "dL_bp": -0.006415441632270813, "ratio": 0.7636836200709354}}, {"step": 3100, "epoch": 6, "train_loss": 0.027722256258130074, "cos_r_negg": [0.4568433463573456, 0.4942241907119751, 0.5254814624786377], "cos_innovation_negg": [0.4568433463573456, 0.4942241907119751, 0.5254814624786377], "cos_apical_negg": [0.018839817494153976, 0.0616610124707222, 0.12520012259483337], "cos_Ac_negg": [0.5735263824462891, 0.6091631650924683, 0.6174659729003906], "r_norm": [0.1963949054479599, 0.1905369758605957, 0.1898682564496994]}, {"step": 3200, "epoch": 6, "train_loss": 0.028760284185409546, "cos_r_negg": [0.451200008392334, 0.4926733374595642, 0.5232774019241333], "cos_innovation_negg": [0.451200008392334, 0.4926733374595642, 0.5232774019241333], "cos_apical_negg": [0.008969509974122047, 0.05782996863126755, 0.12332116812467575], "cos_Ac_negg": [0.571612536907196, 0.614120602607727, 0.625178337097168], "r_norm": [0.20188534259796143, 0.19708651304244995, 0.19598671793937683]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.972, "test_loss": 0.1148906593322754}, {"step": 3300, "epoch": 7, "train_loss": 0.016973750665783882, "cos_r_negg": [0.42888927459716797, 0.47589051723480225, 0.5042517781257629], "cos_innovation_negg": [0.42888927459716797, 0.47589051723480225, 0.5042517781257629], "cos_apical_negg": [0.010451404377818108, 0.05136895179748535, 0.11547611653804779], "cos_Ac_negg": [0.5732630491256714, 0.616134524345398, 0.6202009916305542], "r_norm": [0.1492937207221985, 0.146323561668396, 0.1439751237630844]}, {"step": 3400, "epoch": 7, "train_loss": 0.02213393524289131, "cos_r_negg": [0.43600428104400635, 0.47114014625549316, 0.5057457685470581], "cos_innovation_negg": [0.43600428104400635, 0.47114014625549316, 0.5057457685470581], "cos_apical_negg": [0.013777820393443108, 0.05915515869855881, 0.11904092133045197], "cos_Ac_negg": [0.5828040838241577, 0.6167933940887451, 0.6320329904556274], "r_norm": [0.19583429396152496, 0.19101858139038086, 0.18872876465320587]}, {"step": 3500, "epoch": 7, "train_loss": 0.002648072550073266, "cos_r_negg": [0.41043397784233093, 0.4439479410648346, 0.48292601108551025], "cos_innovation_negg": [0.41043397784233093, 0.4439479410648346, 0.48292601108551025], "cos_apical_negg": [0.023740924894809723, 0.053989872336387634, 0.12198619544506073], "cos_Ac_negg": [0.5743815898895264, 0.6118054389953613, 0.6281580328941345], "r_norm": [0.17663255333900452, 0.17193827033042908, 0.1708666831254959], "ldr": {"dL_sdil": -0.004703693091869354, "dL_bp": -0.006071396172046661, "ratio": 0.7747300552590598}}, {"step": 3600, "epoch": 7, "train_loss": 0.0039043196011334658, "cos_r_negg": [0.4013591706752777, 0.44087207317352295, 0.4851040840148926], "cos_innovation_negg": [0.4013591706752777, 0.44087207317352295, 0.4851040840148926], "cos_apical_negg": [0.004526400938630104, 0.05360887572169304, 0.13372525572776794], "cos_Ac_negg": [0.585268497467041, 0.6179358959197998, 0.6315387487411499], "r_norm": [0.2155553251504898, 0.2096172571182251, 0.20781230926513672]}, {"step": 3700, "epoch": 7, "train_loss": 0.009248893707990646, "cos_r_negg": [0.3922937512397766, 0.4375448226928711, 0.4840736985206604], "cos_innovation_negg": [0.3922937512397766, 0.4375448226928711, 0.4840736985206604], "cos_apical_negg": [0.018970292061567307, 0.04961482435464859, 0.13202771544456482], "cos_Ac_negg": [0.5691043138504028, 0.6164520382881165, 0.6411965489387512], "r_norm": [0.17202125489711761, 0.16713252663612366, 0.16515932977199554]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9719, "test_loss": 0.11075621185302735}, {"step": 3800, "epoch": 8, "train_loss": 0.004981127567589283, "cos_r_negg": [0.38537728786468506, 0.421830415725708, 0.47898876667022705], "cos_innovation_negg": [0.38537728786468506, 0.421830415725708, 0.47898876667022705], "cos_apical_negg": [0.015070710331201553, 0.04315692558884621, 0.12926532328128815], "cos_Ac_negg": [0.5603194236755371, 0.6112616062164307, 0.6418333053588867], "r_norm": [0.1709100604057312, 0.16723929345607758, 0.1659994125366211]}, {"step": 3900, "epoch": 8, "train_loss": 0.03475186601281166, "cos_r_negg": [0.3460403084754944, 0.39389103651046753, 0.44235631823539734], "cos_innovation_negg": [0.3460403084754944, 0.39389103651046753, 0.44235631823539734], "cos_apical_negg": [0.005961806513369083, 0.048433251678943634, 0.12342236936092377], "cos_Ac_negg": [0.5594178438186646, 0.6090369820594788, 0.6436883211135864], "r_norm": [0.1878441870212555, 0.18313905596733093, 0.18262165784835815]}, {"step": 4000, "epoch": 8, "train_loss": 0.011655304580926895, "cos_r_negg": [0.33568960428237915, 0.3788948655128479, 0.4304720163345337], "cos_innovation_negg": [0.33568960428237915, 0.3788948655128479, 0.4304720163345337], "cos_apical_negg": [0.003411297220736742, 0.04439641535282135, 0.11890177428722382], "cos_Ac_negg": [0.5630279779434204, 0.6104016304016113, 0.6430118083953857], "r_norm": [0.15784889459609985, 0.15452316403388977, 0.15459957718849182], "ldr": {"dL_sdil": -0.005230657756328583, "dL_bp": -0.007455557584762573, "ratio": 0.7015783456650957}}, {"step": 4100, "epoch": 8, "train_loss": 0.019469017162919044, "cos_r_negg": [0.3555900454521179, 0.3879237174987793, 0.44087815284729004], "cos_innovation_negg": [0.3555900454521179, 0.3879237174987793, 0.44087815284729004], "cos_apical_negg": [0.010583958588540554, 0.0442170724272728, 0.11984287202358246], "cos_Ac_negg": [0.5757404565811157, 0.6180993318557739, 0.6518227458000183], "r_norm": [0.15780150890350342, 0.1544812172651291, 0.1525699347257614]}, {"step": 4200, "epoch": 8, "train_loss": 0.008531911298632622, "cos_r_negg": [0.3404672145843506, 0.3671143651008606, 0.41969335079193115], "cos_innovation_negg": [0.3404672145843506, 0.3671143651008606, 0.41969335079193115], "cos_apical_negg": [0.006818513385951519, 0.04619338735938072, 0.11950722336769104], "cos_Ac_negg": [0.5778594017028809, 0.609337568283081, 0.6469215154647827], "r_norm": [0.18648594617843628, 0.1825118064880371, 0.18113794922828674]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9728, "test_loss": 0.12154939975738525}, {"step": 4300, "epoch": 9, "train_loss": 0.002347099594771862, "cos_r_negg": [0.30231285095214844, 0.3445425033569336, 0.39256227016448975], "cos_innovation_negg": [0.30231285095214844, 0.3445425033569336, 0.39256227016448975], "cos_apical_negg": [0.006321145221590996, 0.04767496883869171, 0.11192117631435394], "cos_Ac_negg": [0.5709518194198608, 0.6096788048744202, 0.6530248522758484], "r_norm": [0.18004506826400757, 0.17584103345870972, 0.17429956793785095]}, {"step": 4400, "epoch": 9, "train_loss": 0.0020861858502030373, "cos_r_negg": [0.3197347819805145, 0.3601011037826538, 0.4065506160259247], "cos_innovation_negg": [0.3197347819805145, 0.3601011037826538, 0.4065506160259247], "cos_apical_negg": [0.011133786290884018, 0.04215654358267784, 0.11003465950489044], "cos_Ac_negg": [0.5672476887702942, 0.6136364936828613, 0.655921459197998], "r_norm": [0.17637914419174194, 0.17255297303199768, 0.1706320345401764]}, {"step": 4500, "epoch": 9, "train_loss": 0.08592142164707184, "cos_r_negg": [0.2888733446598053, 0.3300521969795227, 0.38369232416152954], "cos_innovation_negg": [0.2888733446598053, 0.3300521969795227, 0.38369232416152954], "cos_apical_negg": [0.004504508338868618, 0.0392308235168457, 0.10850073397159576], "cos_Ac_negg": [0.5656898021697998, 0.6128201484680176, 0.6578087210655212], "r_norm": [0.15408459305763245, 0.15064546465873718, 0.15010976791381836], "ldr": {"dL_sdil": -0.006333485245704651, "dL_bp": -0.009062185883522034, "ratio": 0.6988915618273691}}, {"step": 4600, "epoch": 9, "train_loss": 0.0007154761697165668, "cos_r_negg": [0.3099958896636963, 0.35218751430511475, 0.4034431576728821], "cos_innovation_negg": [0.3099958896636963, 0.35218751430511475, 0.4034431576728821], "cos_apical_negg": [0.007443415932357311, 0.04824552685022354, 0.11563584953546524], "cos_Ac_negg": [0.574338436126709, 0.6171183586120605, 0.665877103805542], "r_norm": [0.17084619402885437, 0.1666598618030548, 0.16515648365020752]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9732, "test_loss": 0.12222069816589355}, {"step": 4700, "epoch": 10, "train_loss": 0.0026040682569146156, "cos_r_negg": [0.29173386096954346, 0.3377118706703186, 0.38743066787719727], "cos_innovation_negg": [0.29173386096954346, 0.3377118706703186, 0.38743066787719727], "cos_apical_negg": [0.00429872889071703, 0.04466613754630089, 0.1074349582195282], "cos_Ac_negg": [0.5661612749099731, 0.6161507368087769, 0.6612042188644409], "r_norm": [0.1724032461643219, 0.16908012330532074, 0.16827446222305298]}, {"step": 4800, "epoch": 10, "train_loss": 0.001310874940827489, "cos_r_negg": [0.2771848440170288, 0.31955569982528687, 0.3658789396286011], "cos_innovation_negg": [0.2771848440170288, 0.31955569982528687, 0.3658789396286011], "cos_apical_negg": [0.011703872121870518, 0.04343767464160919, 0.10773435235023499], "cos_Ac_negg": [0.562920331954956, 0.6150481700897217, 0.6620797514915466], "r_norm": [0.19228512048721313, 0.18825951218605042, 0.18735802173614502]}, {"step": 4900, "epoch": 10, "train_loss": 0.0020039535593241453, "cos_r_negg": [0.27285924553871155, 0.30626723170280457, 0.3549470901489258], "cos_innovation_negg": [0.27285924553871155, 0.30626723170280457, 0.3549470901489258], "cos_apical_negg": [0.009632485918700695, 0.04014107584953308, 0.10557032376527786], "cos_Ac_negg": [0.5647563338279724, 0.6165950298309326, 0.6573599576950073], "r_norm": [0.17164576053619385, 0.16816885769367218, 0.16720114648342133]}, {"step": 5000, "epoch": 10, "train_loss": 0.0003416362451389432, "cos_r_negg": [0.27346372604370117, 0.3087601661682129, 0.353363037109375], "cos_innovation_negg": [0.27346372604370117, 0.3087601661682129, 0.353363037109375], "cos_apical_negg": [0.010231495834887028, 0.041025158017873764, 0.10133885592222214], "cos_Ac_negg": [0.5709128379821777, 0.6157529354095459, 0.6610754728317261], "r_norm": [0.17473262548446655, 0.1712895780801773, 0.169683039188385], "ldr": {"dL_sdil": -0.00790444016456604, "dL_bp": -0.014965161681175232, "ratio": 0.5281894264135538}}, {"step": 5100, "epoch": 10, "train_loss": 0.0032001868821680546, "cos_r_negg": [0.2911231517791748, 0.32749468088150024, 0.3695777356624603], "cos_innovation_negg": [0.2911231517791748, 0.32749468088150024, 0.3695777356624603], "cos_apical_negg": [0.00820685364305973, 0.04461097717285156, 0.10714161396026611], "cos_Ac_negg": [0.5806765556335449, 0.6277589797973633, 0.6685709953308105], "r_norm": [0.18072524666786194, 0.1775861382484436, 0.17636726796627045]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9745, "test_loss": 0.1253031810760498}, {"step": 5200, "epoch": 11, "train_loss": 0.0013817797880619764, "cos_r_negg": [0.2580982744693756, 0.2934572696685791, 0.3376128077507019], "cos_innovation_negg": [0.2580982744693756, 0.2934572696685791, 0.3376128077507019], "cos_apical_negg": [0.0053029730916023254, 0.03961411118507385, 0.09642277657985687], "cos_Ac_negg": [0.5685118436813354, 0.6207524538040161, 0.6629378795623779], "r_norm": [0.16845902800559998, 0.16551652550697327, 0.16457808017730713]}, {"step": 5300, "epoch": 11, "train_loss": 0.0031235001515597105, "cos_r_negg": [0.2628600299358368, 0.2962356507778168, 0.33987027406692505], "cos_innovation_negg": [0.2628600299358368, 0.2962356507778168, 0.33987027406692505], "cos_apical_negg": [0.010412672534584999, 0.040117181837558746, 0.09692540764808655], "cos_Ac_negg": [0.5716552734375, 0.6213104724884033, 0.6641717553138733], "r_norm": [0.17262420058250427, 0.1692957580089569, 0.1677445024251938]}, {"step": 5400, "epoch": 11, "train_loss": 0.002204406075179577, "cos_r_negg": [0.25126463174819946, 0.2824426591396332, 0.32608067989349365], "cos_innovation_negg": [0.25126463174819946, 0.2824426591396332, 0.32608067989349365], "cos_apical_negg": [0.010494066402316093, 0.03892509266734123, 0.09454195201396942], "cos_Ac_negg": [0.5691818594932556, 0.6231475472450256, 0.6644560098648071], "r_norm": [0.16190442442893982, 0.15893208980560303, 0.15774022042751312]}, {"step": 5500, "epoch": 11, "train_loss": 0.0012879519490525126, "cos_r_negg": [0.24600161612033844, 0.2796240448951721, 0.32071489095687866], "cos_innovation_negg": [0.24600161612033844, 0.2796240448951721, 0.32071489095687866], "cos_apical_negg": [0.009254278615117073, 0.03623935580253601, 0.09265763312578201], "cos_Ac_negg": [0.568467378616333, 0.6219388246536255, 0.666171669960022], "r_norm": [0.17348966002464294, 0.17073673009872437, 0.16944527626037598], "ldr": {"dL_sdil": -0.00840149074792862, "dL_bp": -0.015048593282699585, "ratio": 0.5582907711106314}}, {"step": 5600, "epoch": 11, "train_loss": 0.000376337964553386, "cos_r_negg": [0.23778128623962402, 0.27161866426467896, 0.3129546046257019], "cos_innovation_negg": [0.23778128623962402, 0.27161866426467896, 0.3129546046257019], "cos_apical_negg": [0.00874683540314436, 0.03531908243894577, 0.09104202687740326], "cos_Ac_negg": [0.5639955401420593, 0.6225323677062988, 0.6651027202606201], "r_norm": [0.16939803957939148, 0.16641424596309662, 0.16531902551651]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9737, "test_loss": 0.12698190479278565}, {"step": 5700, "epoch": 12, "train_loss": 0.0006420986610464752, "cos_r_negg": [0.23718231916427612, 0.27198225259780884, 0.3126005530357361], "cos_innovation_negg": [0.23718231916427612, 0.27198225259780884, 0.3126005530357361], "cos_apical_negg": [0.007774059660732746, 0.037944331765174866, 0.09066498279571533], "cos_Ac_negg": [0.5632534027099609, 0.6217111349105835, 0.663850724697113], "r_norm": [0.16592973470687866, 0.1630379855632782, 0.16169889271259308]}, {"step": 5800, "epoch": 12, "train_loss": 0.0007728916243650019, "cos_r_negg": [0.22984443604946136, 0.2670248746871948, 0.3072122633457184], "cos_innovation_negg": [0.22984443604946136, 0.2670248746871948, 0.3072122633457184], "cos_apical_negg": [0.006772289052605629, 0.03654114156961441, 0.0885317474603653], "cos_Ac_negg": [0.5602132081985474, 0.6207728981971741, 0.662926197052002], "r_norm": [0.1717938482761383, 0.16880059242248535, 0.1673453003168106]}, {"step": 5900, "epoch": 12, "train_loss": 0.0007180860266089439, "cos_r_negg": [0.2311168909072876, 0.2640523612499237, 0.3034852147102356], "cos_innovation_negg": [0.2311168909072876, 0.2640523612499237, 0.3034852147102356], "cos_apical_negg": [0.009076560847461224, 0.037591107189655304, 0.08804185688495636], "cos_Ac_negg": [0.5626230239868164, 0.6205068826675415, 0.6645230650901794], "r_norm": [0.16732507944107056, 0.1644558310508728, 0.1630568504333496]}, {"step": 6000, "epoch": 12, "train_loss": 0.0004005081136710942, "cos_r_negg": [0.22908103466033936, 0.25990188121795654, 0.2995113730430603], "cos_innovation_negg": [0.22908103466033936, 0.25990188121795654, 0.2995113730430603], "cos_apical_negg": [0.009052458219230175, 0.03492584079504013, 0.08742804080247879], "cos_Ac_negg": [0.5630617141723633, 0.6191227436065674, 0.6655107140541077], "r_norm": [0.17087136209011078, 0.1677241027355194, 0.16645754873752594], "ldr": {"dL_sdil": -0.00819569081068039, "dL_bp": -0.015367090702056885, "ratio": 0.5333274182850627}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9741, "test_loss": 0.13171885814666748}, {"step": 6100, "epoch": 13, "train_loss": 0.001223524333909154, "cos_r_negg": [0.22356580197811127, 0.2555235028266907, 0.2955303192138672], "cos_innovation_negg": [0.22356580197811127, 0.2555235028266907, 0.2955303192138672], "cos_apical_negg": [0.006699959747493267, 0.03435293585062027, 0.08562125265598297], "cos_Ac_negg": [0.5636541247367859, 0.6195017695426941, 0.6660257577896118], "r_norm": [0.16979342699050903, 0.1667345017194748, 0.1653338074684143]}, {"step": 6200, "epoch": 13, "train_loss": 0.0006305642891675234, "cos_r_negg": [0.21781311929225922, 0.2522279620170593, 0.2905955910682678], "cos_innovation_negg": [0.21781311929225922, 0.2522279620170593, 0.2905955910682678], "cos_apical_negg": [0.006162715144455433, 0.03480545058846474, 0.08396048098802567], "cos_Ac_negg": [0.5646047592163086, 0.6194910407066345, 0.6657593250274658], "r_norm": [0.16934242844581604, 0.16631655395030975, 0.16489391028881073]}, {"step": 6300, "epoch": 13, "train_loss": 0.0010061676148325205, "cos_r_negg": [0.2125268280506134, 0.24801746010780334, 0.28590768575668335], "cos_innovation_negg": [0.2125268280506134, 0.24801746010780334, 0.28590768575668335], "cos_apical_negg": [0.0057232566177845, 0.034615419805049896, 0.08182793855667114], "cos_Ac_negg": [0.5633645057678223, 0.6183228492736816, 0.6638625860214233], "r_norm": [0.17359843850135803, 0.17037592828273773, 0.16925080120563507]}, {"step": 6400, "epoch": 13, "train_loss": 0.0006444831960834563, "cos_r_negg": [0.21569207310676575, 0.2516207695007324, 0.28888556361198425], "cos_innovation_negg": [0.21569207310676575, 0.2516207695007324, 0.28888556361198425], "cos_apical_negg": [0.005531219765543938, 0.03592689335346222, 0.08304287493228912], "cos_Ac_negg": [0.5640223622322083, 0.6188873052597046, 0.6671523451805115], "r_norm": [0.1723746508359909, 0.1693345606327057, 0.16798320412635803]}, {"step": 6500, "epoch": 13, "train_loss": 0.00018717575585469604, "cos_r_negg": [0.21536779403686523, 0.24663840234279633, 0.2857993245124817], "cos_innovation_negg": [0.21536779403686523, 0.24663840234279633, 0.2857993245124817], "cos_apical_negg": [0.007480944041162729, 0.034892238676548004, 0.08446259051561356], "cos_Ac_negg": [0.5645676255226135, 0.6208353638648987, 0.6680217981338501], "r_norm": [0.1689678132534027, 0.1657947301864624, 0.1644662618637085], "ldr": {"dL_sdil": -0.00848175585269928, "dL_bp": -0.01608678698539734, "ratio": 0.5272498392872691}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.974, "test_loss": 0.13420242557525636}, {"step": 6600, "epoch": 14, "train_loss": 0.0003008057246915996, "cos_r_negg": [0.20793631672859192, 0.24171459674835205, 0.27859508991241455], "cos_innovation_negg": [0.20793631672859192, 0.24171459674835205, 0.27859508991241455], "cos_apical_negg": [0.0067939795553684235, 0.035257384181022644, 0.08083803951740265], "cos_Ac_negg": [0.5628925561904907, 0.6220639944076538, 0.6661484241485596], "r_norm": [0.17030203342437744, 0.16713452339172363, 0.16584137082099915]}, {"step": 6700, "epoch": 14, "train_loss": 0.0004864076618105173, "cos_r_negg": [0.20941156148910522, 0.2427048683166504, 0.28143978118896484], "cos_innovation_negg": [0.20941156148910522, 0.2427048683166504, 0.28143978118896484], "cos_apical_negg": [0.005215403623878956, 0.035642109811306, 0.08290765434503555], "cos_Ac_negg": [0.5644778609275818, 0.6189313530921936, 0.6686806678771973], "r_norm": [0.17117127776145935, 0.168001189827919, 0.16660992801189423]}, {"step": 6800, "epoch": 14, "train_loss": 0.000661215279251337, "cos_r_negg": [0.20596365630626678, 0.23940075933933258, 0.2752740681171417], "cos_innovation_negg": [0.20596365630626678, 0.23940075933933258, 0.2752740681171417], "cos_apical_negg": [0.005828985013067722, 0.034926820546388626, 0.07993684709072113], "cos_Ac_negg": [0.5642367601394653, 0.6217203140258789, 0.6673427820205688], "r_norm": [0.167779803276062, 0.1646818220615387, 0.16323944926261902]}, {"step": 6900, "epoch": 14, "train_loss": 0.0004742747114505619, "cos_r_negg": [0.19966554641723633, 0.2355295568704605, 0.271500825881958], "cos_innovation_negg": [0.19966554641723633, 0.2355295568704605, 0.271500825881958], "cos_apical_negg": [0.006272670812904835, 0.03522138670086861, 0.07889480888843536], "cos_Ac_negg": [0.5620257258415222, 0.6227530241012573, 0.6663877367973328], "r_norm": [0.16830317676067352, 0.1651759147644043, 0.16383156180381775]}, {"step": 7000, "epoch": 14, "train_loss": 0.0005611440283246338, "cos_r_negg": [0.20100966095924377, 0.2357805222272873, 0.27105045318603516], "cos_innovation_negg": [0.20100966095924377, 0.2357805222272873, 0.27105045318603516], "cos_apical_negg": [0.007838251069188118, 0.03555237501859665, 0.0796094536781311], "cos_Ac_negg": [0.5643315315246582, 0.6249253749847412, 0.6667383313179016], "r_norm": [0.1736074686050415, 0.17039860785007477, 0.16925540566444397], "ldr": {"dL_sdil": -0.00937654823064804, "dL_bp": -0.018130972981452942, "ratio": 0.5171563732536456}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9745, "test_loss": 0.13664108638763428}], "final": {"test_acc": 0.9745, "test_loss": 0.13664108638763428, "wall_s": 24.979517221450806, "cos_r_negg": [0.19951781630516052, 0.23315192759037018, 0.26815009117126465], "cos_innovation_negg": [0.19951781630516052, 0.23315192759037018, 0.26815009117126465], "cos_apical_negg": [0.00645437790080905, 0.03431350737810135, 0.0783519297838211], "cos_Ac_negg": [0.5604537129402161, 0.6230258941650391, 0.6663344502449036]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_residual_s1.json b/results/nuis_ctrl_v1_rho0p2_residual_s1.json new file mode 100644 index 0000000..e6ff91a --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_residual_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_residual_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.011931603774428368, -0.013785354793071747, -0.005786198191344738], "cos_innovation_negg": [0.011931603774428368, -0.013785354793071747, -0.005786198191344738], "cos_apical_negg": [0.010163621976971626, -0.012931429781019688, -0.007024887017905712], "cos_Ac_negg": [0.011931432411074638, -0.013785326853394508, -0.005786254070699215], "r_norm": [4.739147186279297, 4.7983622550964355, 4.7884955406188965], "ldr": {"dL_sdil": -0.04240226745605469, "dL_bp": -0.21988940238952637, "ratio": 0.19283452042377447}}, {"step": 100, "epoch": 0, "train_loss": 0.19775736331939697, "cos_r_negg": [0.45872023701667786, 0.3625425696372986, 0.3213750123977661], "cos_innovation_negg": [0.45872023701667786, 0.3625425696372986, 0.3213750123977661], "cos_apical_negg": [0.1356997787952423, 0.10975304245948792, 0.14097663760185242], "cos_Ac_negg": [0.45958253741264343, 0.36344701051712036, 0.32139092683792114], "r_norm": [0.9387720823287964, 0.9419670104980469, 0.9592081904411316]}, {"step": 200, "epoch": 0, "train_loss": 0.32657188177108765, "cos_r_negg": [0.4945112466812134, 0.42765265703201294, 0.3720117211341858], "cos_innovation_negg": [0.4945112466812134, 0.42765265703201294, 0.3720117211341858], "cos_apical_negg": [0.12734103202819824, 0.11078793555498123, 0.14468501508235931], "cos_Ac_negg": [0.49576351046562195, 0.42875152826309204, 0.3724040389060974], "r_norm": [0.7496249079704285, 0.7569105625152588, 0.7658607959747314]}, {"step": 300, "epoch": 0, "train_loss": 0.1660933941602707, "cos_r_negg": [0.5047013759613037, 0.45309486985206604, 0.41172993183135986], "cos_innovation_negg": [0.5047013759613037, 0.45309486985206604, 0.41172993183135986], "cos_apical_negg": [0.10227082669734955, 0.09924061596393585, 0.13187697529792786], "cos_Ac_negg": [0.5056310892105103, 0.4530573785305023, 0.410241961479187], "r_norm": [0.5657568573951721, 0.5680766105651855, 0.5780221819877625]}, {"step": 400, "epoch": 0, "train_loss": 0.1028156578540802, "cos_r_negg": [0.5188640356063843, 0.4748266935348511, 0.4355982542037964], "cos_innovation_negg": [0.5188640356063843, 0.4748266935348511, 0.4355982542037964], "cos_apical_negg": [0.09269370138645172, 0.10672474652528763, 0.14090155065059662], "cos_Ac_negg": [0.5188056826591492, 0.4748728275299072, 0.4347767233848572], "r_norm": [0.5791422128677368, 0.5850809216499329, 0.586473822593689]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9444, "test_loss": 0.18449363555908202}, {"step": 500, "epoch": 1, "train_loss": 0.18749767541885376, "cos_r_negg": [0.5367656946182251, 0.47632521390914917, 0.44773295521736145], "cos_innovation_negg": [0.5367656946182251, 0.47632521390914917, 0.44773295521736145], "cos_apical_negg": [0.08224304020404816, 0.0912894457578659, 0.15145498514175415], "cos_Ac_negg": [0.5402472615242004, 0.47959578037261963, 0.44738397002220154], "r_norm": [0.5153515338897705, 0.518422544002533, 0.5192044377326965], "ldr": {"dL_sdil": -0.013696134090423584, "dL_bp": -0.011990353465080261, "ratio": 1.1422627473252644}}, {"step": 600, "epoch": 1, "train_loss": 0.1040426716208458, "cos_r_negg": [0.530276894569397, 0.48525846004486084, 0.4493897557258606], "cos_innovation_negg": [0.530276894569397, 0.48525846004486084, 0.4493897557258606], "cos_apical_negg": [0.07756748050451279, 0.07605090737342834, 0.1394515335559845], "cos_Ac_negg": [0.5312595367431641, 0.48635751008987427, 0.44880661368370056], "r_norm": [0.4344339072704315, 0.4356266260147095, 0.44007688760757446]}, {"step": 700, "epoch": 1, "train_loss": 0.15639904141426086, "cos_r_negg": [0.5480979681015015, 0.5102795958518982, 0.48049014806747437], "cos_innovation_negg": [0.5480979681015015, 0.5102795958518982, 0.48049014806747437], "cos_apical_negg": [0.07059018313884735, 0.0943416953086853, 0.1362384855747223], "cos_Ac_negg": [0.5526449680328369, 0.5116695165634155, 0.4787350594997406], "r_norm": [0.41975533962249756, 0.424714058637619, 0.4228808283805847]}, {"step": 800, "epoch": 1, "train_loss": 0.16617156565189362, "cos_r_negg": [0.5512282848358154, 0.5170258283615112, 0.47346439957618713], "cos_innovation_negg": [0.5512282848358154, 0.5170258283615112, 0.47346439957618713], "cos_apical_negg": [0.07387018203735352, 0.0937204658985138, 0.12942790985107422], "cos_Ac_negg": [0.5569518804550171, 0.5212909579277039, 0.4727581739425659], "r_norm": [0.4107813537120819, 0.4118606448173523, 0.41608816385269165]}, {"step": 900, "epoch": 1, "train_loss": 0.2058173269033432, "cos_r_negg": [0.5738842487335205, 0.5367188453674316, 0.47506433725357056], "cos_innovation_negg": [0.5738842487335205, 0.5367188453674316, 0.47506433725357056], "cos_apical_negg": [0.061650656163692474, 0.09630943089723587, 0.14210359752178192], "cos_Ac_negg": [0.5736643671989441, 0.5366127490997314, 0.47380971908569336], "r_norm": [0.4241654872894287, 0.4231199622154236, 0.42675256729125977]}, {"epoch_end": 1, "step": 938, "test_acc": 0.955, "test_loss": 0.15527927742004394}, {"step": 1000, "epoch": 2, "train_loss": 0.021358666941523552, "cos_r_negg": [0.5856679081916809, 0.5437238216400146, 0.4949186146259308], "cos_innovation_negg": [0.5856679081916809, 0.5437238216400146, 0.4949186146259308], "cos_apical_negg": [0.046389296650886536, 0.08993111550807953, 0.13293297588825226], "cos_Ac_negg": [0.5890286564826965, 0.5449188351631165, 0.49256759881973267], "r_norm": [0.36267194151878357, 0.36412501335144043, 0.369210809469223], "ldr": {"dL_sdil": -0.009381487965583801, "dL_bp": -0.008346065878868103, "ratio": 1.1240610967782252}}, {"step": 1100, "epoch": 2, "train_loss": 0.11250459402799606, "cos_r_negg": [0.5836842060089111, 0.5534021258354187, 0.5075383186340332], "cos_innovation_negg": [0.5836842060089111, 0.5534021258354187, 0.5075383186340332], "cos_apical_negg": [0.060446105897426605, 0.08326318860054016, 0.12937980890274048], "cos_Ac_negg": [0.5876331329345703, 0.5538390874862671, 0.5071291923522949], "r_norm": [0.37702304124832153, 0.3828517198562622, 0.38200998306274414]}, {"step": 1200, "epoch": 2, "train_loss": 0.13226768374443054, "cos_r_negg": [0.5769443511962891, 0.5558546781539917, 0.5066117644309998], "cos_innovation_negg": [0.5769443511962891, 0.5558546781539917, 0.5066117644309998], "cos_apical_negg": [0.05843479931354523, 0.08167964220046997, 0.11882665008306503], "cos_Ac_negg": [0.58112633228302, 0.5567046999931335, 0.5058470964431763], "r_norm": [0.337144136428833, 0.3376275897026062, 0.34184199571609497]}, {"step": 1300, "epoch": 2, "train_loss": 0.11763103306293488, "cos_r_negg": [0.5968750715255737, 0.5797828435897827, 0.5279954671859741], "cos_innovation_negg": [0.5968750715255737, 0.5797828435897827, 0.5279954671859741], "cos_apical_negg": [0.04198881983757019, 0.0771612599492073, 0.12961767613887787], "cos_Ac_negg": [0.602980375289917, 0.5828290581703186, 0.528378427028656], "r_norm": [0.318722128868103, 0.3170597553253174, 0.3181980848312378]}, {"step": 1400, "epoch": 2, "train_loss": 0.11081075668334961, "cos_r_negg": [0.5979578495025635, 0.5789686441421509, 0.5127284526824951], "cos_innovation_negg": [0.5979578495025635, 0.5789686441421509, 0.5127284526824951], "cos_apical_negg": [0.046362608671188354, 0.07994739711284637, 0.11826172471046448], "cos_Ac_negg": [0.6016546487808228, 0.5812645554542542, 0.5135836601257324], "r_norm": [0.28814074397087097, 0.2862517237663269, 0.29094311594963074]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9643, "test_loss": 0.11845275306701661}, {"step": 1500, "epoch": 3, "train_loss": 0.05337446555495262, "cos_r_negg": [0.5874097347259521, 0.5748370289802551, 0.5361409187316895], "cos_innovation_negg": [0.5874097347259521, 0.5748370289802551, 0.5361409187316895], "cos_apical_negg": [0.04201585799455643, 0.07050016522407532, 0.12474855035543442], "cos_Ac_negg": [0.5923759937286377, 0.5785771608352661, 0.5364388227462769], "r_norm": [0.261452317237854, 0.2613041400909424, 0.2655849754810333], "ldr": {"dL_sdil": -0.005416393280029297, "dL_bp": -0.00526069849729538, "ratio": 1.0295958384260118}}, {"step": 1600, "epoch": 3, "train_loss": 0.05822928994894028, "cos_r_negg": [0.5893609523773193, 0.5796955227851868, 0.5560258626937866], "cos_innovation_negg": [0.5893609523773193, 0.5796955227851868, 0.5560258626937866], "cos_apical_negg": [0.028988126665353775, 0.06752461194992065, 0.1227165162563324], "cos_Ac_negg": [0.5977765917778015, 0.586108922958374, 0.5571414828300476], "r_norm": [0.2562752366065979, 0.2568072974681854, 0.25675591826438904]}, {"step": 1700, "epoch": 3, "train_loss": 0.11532272398471832, "cos_r_negg": [0.5804225206375122, 0.5829477906227112, 0.5600645542144775], "cos_innovation_negg": [0.5804225206375122, 0.5829477906227112, 0.5600645542144775], "cos_apical_negg": [0.041842132806777954, 0.08018238842487335, 0.12040457874536514], "cos_Ac_negg": [0.5935515761375427, 0.5934044122695923, 0.5652104616165161], "r_norm": [0.2907516658306122, 0.28905370831489563, 0.29030248522758484]}, {"step": 1800, "epoch": 3, "train_loss": 0.024315420538187027, "cos_r_negg": [0.596096932888031, 0.5926241278648376, 0.5527631640434265], "cos_innovation_negg": [0.596096932888031, 0.5926241278648376, 0.5527631640434265], "cos_apical_negg": [0.04096980020403862, 0.06974849849939346, 0.11268502473831177], "cos_Ac_negg": [0.6052919626235962, 0.5986686944961548, 0.554040253162384], "r_norm": [0.2584753632545471, 0.258015513420105, 0.2584863305091858]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9632, "test_loss": 0.11869043083190918}, {"step": 1900, "epoch": 4, "train_loss": 0.04801398888230324, "cos_r_negg": [0.5939580798149109, 0.5847367644309998, 0.5548296570777893], "cos_innovation_negg": [0.5939580798149109, 0.5847367644309998, 0.5548296570777893], "cos_apical_negg": [0.06321965157985687, 0.07881684601306915, 0.1235065758228302], "cos_Ac_negg": [0.6099642515182495, 0.5955401062965393, 0.5598971843719482], "r_norm": [0.29030996561050415, 0.2912764251232147, 0.2929227948188782]}, {"step": 2000, "epoch": 4, "train_loss": 0.07748020440340042, "cos_r_negg": [0.5760419368743896, 0.5681276321411133, 0.5477563142776489], "cos_innovation_negg": [0.5760419368743896, 0.5681276321411133, 0.5477563142776489], "cos_apical_negg": [0.02792409248650074, 0.059769026935100555, 0.11725227534770966], "cos_Ac_negg": [0.6076132655143738, 0.5944979190826416, 0.561225414276123], "r_norm": [0.22954419255256653, 0.22847869992256165, 0.2283090054988861], "ldr": {"dL_sdil": -0.0057287514209747314, "dL_bp": -0.006127595901489258, "ratio": 0.9349101202287848}}, {"step": 2100, "epoch": 4, "train_loss": 0.050592273473739624, "cos_r_negg": [0.5596483945846558, 0.574286937713623, 0.5611082315444946], "cos_innovation_negg": [0.5596483945846558, 0.574286937713623, 0.5611082315444946], "cos_apical_negg": [0.024154040962457657, 0.0528561994433403, 0.11123406141996384], "cos_Ac_negg": [0.5905953645706177, 0.600906491279602, 0.5775448679924011], "r_norm": [0.2186432033777237, 0.2179013192653656, 0.22010205686092377]}, {"step": 2200, "epoch": 4, "train_loss": 0.01839619129896164, "cos_r_negg": [0.5920798778533936, 0.5911052227020264, 0.5681278705596924], "cos_innovation_negg": [0.5920798778533936, 0.5911052227020264, 0.5681278705596924], "cos_apical_negg": [0.011098718270659447, 0.05819759517908096, 0.11481077969074249], "cos_Ac_negg": [0.6115607023239136, 0.6076661348342896, 0.575154721736908], "r_norm": [0.21184080839157104, 0.21109355986118317, 0.21288445591926575]}, {"step": 2300, "epoch": 4, "train_loss": 0.16251692175865173, "cos_r_negg": [0.5427846312522888, 0.5610638856887817, 0.5550764203071594], "cos_innovation_negg": [0.5427846312522888, 0.5610638856887817, 0.5550764203071594], "cos_apical_negg": [0.028781840577721596, 0.057624928653240204, 0.11066931486129761], "cos_Ac_negg": [0.5845282077789307, 0.602412223815918, 0.5805932283401489], "r_norm": [0.19856134057044983, 0.19890277087688446, 0.20000869035720825]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.969, "test_loss": 0.11277374114990234}, {"step": 2400, "epoch": 5, "train_loss": 0.07764077931642532, "cos_r_negg": [0.5798075199127197, 0.5902670621871948, 0.579801082611084], "cos_innovation_negg": [0.5798075199127197, 0.5902670621871948, 0.579801082611084], "cos_apical_negg": [0.027051949873566628, 0.04981415718793869, 0.11517209559679031], "cos_Ac_negg": [0.6182402968406677, 0.624508261680603, 0.5985470414161682], "r_norm": [0.20267830789089203, 0.2023141235113144, 0.2052423655986786]}, {"step": 2500, "epoch": 5, "train_loss": 0.03252333402633667, "cos_r_negg": [0.5317485332489014, 0.5525341033935547, 0.5637630820274353], "cos_innovation_negg": [0.5317485332489014, 0.5525341033935547, 0.5637630820274353], "cos_apical_negg": [0.027274269610643387, 0.05618946626782417, 0.12815535068511963], "cos_Ac_negg": [0.6042139530181885, 0.6129974126815796, 0.6021047830581665], "r_norm": [0.2474762499332428, 0.24764174222946167, 0.2503415644168854], "ldr": {"dL_sdil": -0.008180499076843262, "dL_bp": -0.009082622826099396, "ratio": 0.9006758547031334}}, {"step": 2600, "epoch": 5, "train_loss": 0.05673728883266449, "cos_r_negg": [0.5518810749053955, 0.5768418312072754, 0.5871708393096924], "cos_innovation_negg": [0.5518810749053955, 0.5768418312072754, 0.5871708393096924], "cos_apical_negg": [0.010864458978176117, 0.05714278295636177, 0.11639561504125595], "cos_Ac_negg": [0.6136483550071716, 0.6258563995361328, 0.6129207611083984], "r_norm": [0.22200214862823486, 0.2225976437330246, 0.22385293245315552]}, {"step": 2700, "epoch": 5, "train_loss": 0.012749000452458858, "cos_r_negg": [0.5460054278373718, 0.5705655813217163, 0.5896921753883362], "cos_innovation_negg": [0.5460054278373718, 0.5705655813217163, 0.5896921753883362], "cos_apical_negg": [0.020181067287921906, 0.04799661040306091, 0.10917221009731293], "cos_Ac_negg": [0.604796826839447, 0.6227099299430847, 0.6224133968353271], "r_norm": [0.1813088357448578, 0.18050485849380493, 0.18305231630802155]}, {"step": 2800, "epoch": 5, "train_loss": 0.04988821595907211, "cos_r_negg": [0.5203732848167419, 0.555920422077179, 0.5674914717674255], "cos_innovation_negg": [0.5203732848167419, 0.555920422077179, 0.5674914717674255], "cos_apical_negg": [0.026939351111650467, 0.055087488144636154, 0.10572728514671326], "cos_Ac_negg": [0.6057394742965698, 0.6266579627990723, 0.6157872676849365], "r_norm": [0.18579533696174622, 0.18391703069210052, 0.18668165802955627]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9689, "test_loss": 0.11896831932067871}, {"step": 2900, "epoch": 6, "train_loss": 0.006228657439351082, "cos_r_negg": [0.5252091884613037, 0.5586499571800232, 0.5708284378051758], "cos_innovation_negg": [0.5252091884613037, 0.5586499571800232, 0.5708284378051758], "cos_apical_negg": [0.0154110724106431, 0.047098420560359955, 0.11612921953201294], "cos_Ac_negg": [0.6111032962799072, 0.6351197957992554, 0.6190087795257568], "r_norm": [0.182048499584198, 0.18028321862220764, 0.18298307061195374]}, {"step": 3000, "epoch": 6, "train_loss": 0.05967951565980911, "cos_r_negg": [0.503757119178772, 0.5327576398849487, 0.5590969324111938], "cos_innovation_negg": [0.503757119178772, 0.5327576398849487, 0.5590969324111938], "cos_apical_negg": [0.006705448497086763, 0.048671916127204895, 0.11587689816951752], "cos_Ac_negg": [0.6157150864601135, 0.6238573789596558, 0.6224365234375], "r_norm": [0.16073641180992126, 0.15966340899467468, 0.16142268478870392], "ldr": {"dL_sdil": -0.00339449942111969, "dL_bp": -0.004144594073295593, "ratio": 0.8190185482798169}}, {"step": 3100, "epoch": 6, "train_loss": 0.022607536986470222, "cos_r_negg": [0.49863696098327637, 0.51980060338974, 0.5562487244606018], "cos_innovation_negg": [0.49863696098327637, 0.51980060338974, 0.5562487244606018], "cos_apical_negg": [0.00047961785458028316, 0.0421375036239624, 0.10713469982147217], "cos_Ac_negg": [0.6123783588409424, 0.6199117302894592, 0.6267445087432861], "r_norm": [0.16553622484207153, 0.16404928267002106, 0.1662520319223404]}, {"step": 3200, "epoch": 6, "train_loss": 0.03901796042919159, "cos_r_negg": [0.5001592636108398, 0.5136443972587585, 0.5546040534973145], "cos_innovation_negg": [0.5001592636108398, 0.5136443972587585, 0.5546040534973145], "cos_apical_negg": [0.016849977895617485, 0.045684389770030975, 0.11689712107181549], "cos_Ac_negg": [0.6181316375732422, 0.6196607947349548, 0.6289545297622681], "r_norm": [0.17972540855407715, 0.1776731163263321, 0.179264634847641]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9685, "test_loss": 0.12659227161407471}, {"step": 3300, "epoch": 7, "train_loss": 0.0032453604508191347, "cos_r_negg": [0.49487438797950745, 0.5146627426147461, 0.5570589900016785], "cos_innovation_negg": [0.49487438797950745, 0.5146627426147461, 0.5570589900016785], "cos_apical_negg": [0.022871367633342743, 0.05089029669761658, 0.11626814305782318], "cos_Ac_negg": [0.621711015701294, 0.6350054144859314, 0.6384247541427612], "r_norm": [0.18347486853599548, 0.18063265085220337, 0.18405190110206604]}, {"step": 3400, "epoch": 7, "train_loss": 0.04436119645833969, "cos_r_negg": [0.4657537341117859, 0.4919852614402771, 0.5333935022354126], "cos_innovation_negg": [0.4657537341117859, 0.4919852614402771, 0.5333935022354126], "cos_apical_negg": [0.008652392774820328, 0.049738816916942596, 0.11082755029201508], "cos_Ac_negg": [0.6239668130874634, 0.6362565755844116, 0.6375622153282166], "r_norm": [0.18661317229270935, 0.1850159913301468, 0.1869591474533081]}, {"step": 3500, "epoch": 7, "train_loss": 0.006032860372215509, "cos_r_negg": [0.46196189522743225, 0.48990634083747864, 0.5319598317146301], "cos_innovation_negg": [0.46196189522743225, 0.48990634083747864, 0.5319598317146301], "cos_apical_negg": [0.005535840522497892, 0.053613126277923584, 0.1106598973274231], "cos_Ac_negg": [0.6271502375602722, 0.6420409679412842, 0.6415368914604187], "r_norm": [0.18070045113563538, 0.17858397960662842, 0.1823444664478302], "ldr": {"dL_sdil": -0.005282089114189148, "dL_bp": -0.0069892629981040955, "ratio": 0.75574336172812}}, {"step": 3600, "epoch": 7, "train_loss": 0.002231165301054716, "cos_r_negg": [0.44013679027557373, 0.4753730297088623, 0.5182855129241943], "cos_innovation_negg": [0.44013679027557373, 0.4753730297088623, 0.5182855129241943], "cos_apical_negg": [0.005495928227901459, 0.051755014806985855, 0.10805808752775192], "cos_Ac_negg": [0.619712769985199, 0.6374286413192749, 0.6382573843002319], "r_norm": [0.17825168371200562, 0.17635288834571838, 0.17897850275039673]}, {"step": 3700, "epoch": 7, "train_loss": 0.00967714935541153, "cos_r_negg": [0.452364444732666, 0.48681214451789856, 0.5281479358673096], "cos_innovation_negg": [0.452364444732666, 0.48681214451789856, 0.5281479358673096], "cos_apical_negg": [0.0055759381502866745, 0.04553765058517456, 0.11463244259357452], "cos_Ac_negg": [0.6161730289459229, 0.6399081945419312, 0.6419907212257385], "r_norm": [0.19381679594516754, 0.1908852458000183, 0.19416004419326782]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9717, "test_loss": 0.11634278068542481}, {"step": 3800, "epoch": 8, "train_loss": 0.007380823139101267, "cos_r_negg": [0.4137468934059143, 0.45212435722351074, 0.49357855319976807], "cos_innovation_negg": [0.4137468934059143, 0.45212435722351074, 0.49357855319976807], "cos_apical_negg": [0.00708778714761138, 0.05199868232011795, 0.10923279821872711], "cos_Ac_negg": [0.6250726580619812, 0.6390424966812134, 0.6447779536247253], "r_norm": [0.19211000204086304, 0.1904672384262085, 0.1922861933708191]}, {"step": 3900, "epoch": 8, "train_loss": 0.029728081077337265, "cos_r_negg": [0.4061935544013977, 0.43666213750839233, 0.4823722839355469], "cos_innovation_negg": [0.4061935544013977, 0.43666213750839233, 0.4823722839355469], "cos_apical_negg": [0.00817396305501461, 0.04910297691822052, 0.11961860209703445], "cos_Ac_negg": [0.6252659559249878, 0.6392565965652466, 0.6468268632888794], "r_norm": [0.1889357566833496, 0.18662671744823456, 0.18918143212795258]}, {"step": 4000, "epoch": 8, "train_loss": 0.016131335869431496, "cos_r_negg": [0.37239107489585876, 0.4060817062854767, 0.4575170874595642], "cos_innovation_negg": [0.37239107489585876, 0.4060817062854767, 0.4575170874595642], "cos_apical_negg": [9.378185495734215e-05, 0.046725645661354065, 0.11266472935676575], "cos_Ac_negg": [0.6202460527420044, 0.6352185010910034, 0.6429893970489502], "r_norm": [0.17835202813148499, 0.17747706174850464, 0.1794249564409256], "ldr": {"dL_sdil": -0.006239660084247589, "dL_bp": -0.009388342499732971, "ratio": 0.6646178582028789}}, {"step": 4100, "epoch": 8, "train_loss": 0.01696491427719593, "cos_r_negg": [0.40331804752349854, 0.43599748611450195, 0.4769810736179352], "cos_innovation_negg": [0.40331804752349854, 0.43599748611450195, 0.4769810736179352], "cos_apical_negg": [0.0021792910993099213, 0.048161644488573074, 0.10694894939661026], "cos_Ac_negg": [0.6172534823417664, 0.6348395347595215, 0.6427741050720215], "r_norm": [0.18196439743041992, 0.17999687790870667, 0.18242529034614563]}, {"step": 4200, "epoch": 8, "train_loss": 0.006257995031774044, "cos_r_negg": [0.37441813945770264, 0.41094475984573364, 0.45740368962287903], "cos_innovation_negg": [0.37441813945770264, 0.41094475984573364, 0.45740368962287903], "cos_apical_negg": [0.007493656128644943, 0.0479060634970665, 0.10380347073078156], "cos_Ac_negg": [0.610884964466095, 0.6348294019699097, 0.6460144519805908], "r_norm": [0.1809980869293213, 0.17913639545440674, 0.1820543110370636]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9712, "test_loss": 0.12717454872131348}, {"step": 4300, "epoch": 9, "train_loss": 0.0011023624101653695, "cos_r_negg": [0.3483300507068634, 0.382057249546051, 0.43449127674102783], "cos_innovation_negg": [0.3483300507068634, 0.382057249546051, 0.43449127674102783], "cos_apical_negg": [0.008317005820572376, 0.049297407269477844, 0.10036052763462067], "cos_Ac_negg": [0.6123583912849426, 0.6345103979110718, 0.6484857797622681], "r_norm": [0.18181782960891724, 0.17950648069381714, 0.1819377988576889]}, {"step": 4400, "epoch": 9, "train_loss": 0.0008195400005206466, "cos_r_negg": [0.35415226221084595, 0.3898577392101288, 0.4361424744129181], "cos_innovation_negg": [0.35415226221084595, 0.3898577392101288, 0.4361424744129181], "cos_apical_negg": [0.0060727475211024284, 0.04846321791410446, 0.10123865306377411], "cos_Ac_negg": [0.614793062210083, 0.6358656883239746, 0.648384690284729], "r_norm": [0.17908819019794464, 0.17724967002868652, 0.1796587109565735]}, {"step": 4500, "epoch": 9, "train_loss": 0.010645082220435143, "cos_r_negg": [0.33410248160362244, 0.36796867847442627, 0.4165004789829254], "cos_innovation_negg": [0.33410248160362244, 0.36796867847442627, 0.4165004789829254], "cos_apical_negg": [0.007590829860419035, 0.048557184636592865, 0.10064555704593658], "cos_Ac_negg": [0.6158840656280518, 0.6369221210479736, 0.6506656408309937], "r_norm": [0.18416178226470947, 0.18281283974647522, 0.1849346160888672], "ldr": {"dL_sdil": -0.007865674793720245, "dL_bp": -0.013627573847770691, "ratio": 0.5771881981037275}}, {"step": 4600, "epoch": 9, "train_loss": 0.001619409304112196, "cos_r_negg": [0.33882132172584534, 0.37668660283088684, 0.4232252240180969], "cos_innovation_negg": [0.33882132172584534, 0.37668660283088684, 0.4232252240180969], "cos_apical_negg": [0.008896243758499622, 0.05055942013859749, 0.10552978515625], "cos_Ac_negg": [0.6186201572418213, 0.6367286443710327, 0.649141788482666], "r_norm": [0.1943180114030838, 0.1918736696243286, 0.19443336129188538]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9721, "test_loss": 0.12973360595703126}, {"step": 4700, "epoch": 10, "train_loss": 0.0021977010183036327, "cos_r_negg": [0.32870155572891235, 0.3676977753639221, 0.4149904251098633], "cos_innovation_negg": [0.32870155572891235, 0.3676977753639221, 0.4149904251098633], "cos_apical_negg": [0.0030493647791445255, 0.05364247411489487, 0.10174606740474701], "cos_Ac_negg": [0.6212400794029236, 0.6417281627655029, 0.6570532321929932], "r_norm": [0.1943536400794983, 0.19208161532878876, 0.19493183493614197]}, {"step": 4800, "epoch": 10, "train_loss": 0.0048472220078110695, "cos_r_negg": [0.3143385052680969, 0.3508572280406952, 0.40000349283218384], "cos_innovation_negg": [0.3143385052680969, 0.3508572280406952, 0.40000349283218384], "cos_apical_negg": [0.0022430382668972015, 0.050690457224845886, 0.10255388915538788], "cos_Ac_negg": [0.6173672676086426, 0.6412702798843384, 0.6560332179069519], "r_norm": [0.19109372794628143, 0.18872100114822388, 0.1914520412683487]}, {"step": 4900, "epoch": 10, "train_loss": 0.0014961820561438799, "cos_r_negg": [0.3018723726272583, 0.3383164405822754, 0.3870891332626343], "cos_innovation_negg": [0.3018723726272583, 0.3383164405822754, 0.3870891332626343], "cos_apical_negg": [-0.002491031074896455, 0.049107689410448074, 0.10321295261383057], "cos_Ac_negg": [0.6167256832122803, 0.6392117738723755, 0.655708372592926], "r_norm": [0.19168445467948914, 0.18950015306472778, 0.19210729002952576]}, {"step": 5000, "epoch": 10, "train_loss": 0.0008151744259521365, "cos_r_negg": [0.2916484475135803, 0.33086228370666504, 0.3762313425540924], "cos_innovation_negg": [0.2916484475135803, 0.33086228370666504, 0.3762313425540924], "cos_apical_negg": [-0.005651507060974836, 0.05090652406215668, 0.0991320013999939], "cos_Ac_negg": [0.6159945726394653, 0.6375445127487183, 0.6598638892173767], "r_norm": [0.18478374183177948, 0.1824667751789093, 0.18539118766784668], "ldr": {"dL_sdil": -0.008957244455814362, "dL_bp": -0.016509532928466797, "ratio": 0.5425498404239956}}, {"step": 5100, "epoch": 10, "train_loss": 0.006166377104818821, "cos_r_negg": [0.29230961203575134, 0.3324953317642212, 0.3814157247543335], "cos_innovation_negg": [0.29230961203575134, 0.3324953317642212, 0.3814157247543335], "cos_apical_negg": [-0.008434660732746124, 0.0491769015789032, 0.10000497102737427], "cos_Ac_negg": [0.6172416806221008, 0.6376235485076904, 0.6609718799591064], "r_norm": [0.17606347799301147, 0.1740131825208664, 0.17657995223999023]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9715, "test_loss": 0.12970616302490234}, {"step": 5200, "epoch": 11, "train_loss": 0.0005540874553844333, "cos_r_negg": [0.29465028643608093, 0.33385953307151794, 0.3777812421321869], "cos_innovation_negg": [0.29465028643608093, 0.33385953307151794, 0.3777812421321869], "cos_apical_negg": [-0.0082553057000041, 0.04947355017066002, 0.09809713065624237], "cos_Ac_negg": [0.6166052222251892, 0.6384375691413879, 0.6590942144393921], "r_norm": [0.17594361305236816, 0.17453598976135254, 0.1756635457277298]}, {"step": 5300, "epoch": 11, "train_loss": 0.0010943440720438957, "cos_r_negg": [0.28236421942710876, 0.3227180242538452, 0.36728060245513916], "cos_innovation_negg": [0.28236421942710876, 0.3227180242538452, 0.36728060245513916], "cos_apical_negg": [-0.008325804956257343, 0.048340387642383575, 0.09489801526069641], "cos_Ac_negg": [0.609477162361145, 0.6339924931526184, 0.6580001711845398], "r_norm": [0.17443129420280457, 0.1728535145521164, 0.17435342073440552]}, {"step": 5400, "epoch": 11, "train_loss": 0.0020020240917801857, "cos_r_negg": [0.27028995752334595, 0.3119666278362274, 0.35435551404953003], "cos_innovation_negg": [0.27028995752334595, 0.3119666278362274, 0.35435551404953003], "cos_apical_negg": [-0.006102823652327061, 0.04947243630886078, 0.09166481345891953], "cos_Ac_negg": [0.6115628480911255, 0.6343180537223816, 0.6585925221443176], "r_norm": [0.18084609508514404, 0.17881640791893005, 0.18106326460838318]}, {"step": 5500, "epoch": 11, "train_loss": 0.0023182944860309362, "cos_r_negg": [0.26846668124198914, 0.31433433294296265, 0.3568243980407715], "cos_innovation_negg": [0.26846668124198914, 0.31433433294296265, 0.3568243980407715], "cos_apical_negg": [-0.009181167930364609, 0.050728313624858856, 0.09624015539884567], "cos_Ac_negg": [0.6113748550415039, 0.6360570192337036, 0.6586302518844604], "r_norm": [0.1936633437871933, 0.19143393635749817, 0.19355353713035583], "ldr": {"dL_sdil": -0.009663648903369904, "dL_bp": -0.018513984978199005, "ratio": 0.5219648236049266}}, {"step": 5600, "epoch": 11, "train_loss": 0.0003160181222483516, "cos_r_negg": [0.2535359859466553, 0.29683980345726013, 0.33718693256378174], "cos_innovation_negg": [0.2535359859466553, 0.29683980345726013, 0.33718693256378174], "cos_apical_negg": [-0.009216856211423874, 0.04909816384315491, 0.0910632386803627], "cos_Ac_negg": [0.6061104536056519, 0.6344951391220093, 0.6641950011253357], "r_norm": [0.18347963690757751, 0.18202190101146698, 0.1838439702987671]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9727, "test_loss": 0.13577726669311524}, {"step": 5700, "epoch": 12, "train_loss": 0.000947620952501893, "cos_r_negg": [0.25386619567871094, 0.2974931001663208, 0.3395717144012451], "cos_innovation_negg": [0.25386619567871094, 0.2974931001663208, 0.3395717144012451], "cos_apical_negg": [-0.01228018943220377, 0.04886765405535698, 0.09168899059295654], "cos_Ac_negg": [0.6076353788375854, 0.6335341930389404, 0.6627957820892334], "r_norm": [0.1760621815919876, 0.1745014786720276, 0.17572079598903656]}, {"step": 5800, "epoch": 12, "train_loss": 0.00036313594318926334, "cos_r_negg": [0.24559952318668365, 0.2888130843639374, 0.33031395077705383], "cos_innovation_negg": [0.24559952318668365, 0.2888130843639374, 0.33031395077705383], "cos_apical_negg": [-0.013709278777241707, 0.04781469330191612, 0.0893857330083847], "cos_Ac_negg": [0.6113327741622925, 0.6327759027481079, 0.6631534099578857], "r_norm": [0.18037748336791992, 0.17876042425632477, 0.18040570616722107]}, {"step": 5900, "epoch": 12, "train_loss": 0.003891424275934696, "cos_r_negg": [0.24846214056015015, 0.2900077998638153, 0.33367830514907837], "cos_innovation_negg": [0.24846214056015015, 0.2900077998638153, 0.33367830514907837], "cos_apical_negg": [-0.010402785614132881, 0.04911920055747032, 0.09120796620845795], "cos_Ac_negg": [0.6064143776893616, 0.6320850849151611, 0.6621131896972656], "r_norm": [0.1823580116033554, 0.1804198920726776, 0.1822509765625]}, {"step": 6000, "epoch": 12, "train_loss": 0.00027683001826517284, "cos_r_negg": [0.24804182350635529, 0.2855919599533081, 0.3256784677505493], "cos_innovation_negg": [0.24804182350635529, 0.2855919599533081, 0.3256784677505493], "cos_apical_negg": [-0.008538182824850082, 0.048475660383701324, 0.08708613365888596], "cos_Ac_negg": [0.6110955476760864, 0.6319037675857544, 0.6617473363876343], "r_norm": [0.17901301383972168, 0.17723117768764496, 0.17906996607780457], "ldr": {"dL_sdil": -0.009073600172996521, "dL_bp": -0.01850360631942749, "ratio": 0.4903692834985295}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9729, "test_loss": 0.13838680152893065}, {"step": 6100, "epoch": 13, "train_loss": 0.0006825184100307524, "cos_r_negg": [0.24415338039398193, 0.28162163496017456, 0.32339686155319214], "cos_innovation_negg": [0.24415338039398193, 0.28162163496017456, 0.32339686155319214], "cos_apical_negg": [-0.004759262781590223, 0.04672665894031525, 0.08735356479883194], "cos_Ac_negg": [0.6095070838928223, 0.630984902381897, 0.6619540452957153], "r_norm": [0.18518301844596863, 0.18320360779762268, 0.1852014660835266]}, {"step": 6200, "epoch": 13, "train_loss": 0.0008595471736043692, "cos_r_negg": [0.23484259843826294, 0.2732177972793579, 0.3125447928905487], "cos_innovation_negg": [0.23484259843826294, 0.2732177972793579, 0.3125447928905487], "cos_apical_negg": [-0.0061957696452736855, 0.04641520231962204, 0.08374373614788055], "cos_Ac_negg": [0.606260359287262, 0.6301473379135132, 0.6641076803207397], "r_norm": [0.18128171563148499, 0.17957058548927307, 0.1814786195755005]}, {"step": 6300, "epoch": 13, "train_loss": 0.0006934639532119036, "cos_r_negg": [0.2264966070652008, 0.26610812544822693, 0.30659109354019165], "cos_innovation_negg": [0.2264966070652008, 0.26610812544822693, 0.30659109354019165], "cos_apical_negg": [-0.008863432332873344, 0.04641268402338028, 0.08453816175460815], "cos_Ac_negg": [0.6030253171920776, 0.626876950263977, 0.6629367470741272], "r_norm": [0.17757649719715118, 0.17598430812358856, 0.17771857976913452]}, {"step": 6400, "epoch": 13, "train_loss": 0.0002877316437661648, "cos_r_negg": [0.23091086745262146, 0.2681494355201721, 0.3060353994369507], "cos_innovation_negg": [0.23091086745262146, 0.2681494355201721, 0.3060353994369507], "cos_apical_negg": [-0.006315913982689381, 0.04519297927618027, 0.08115345239639282], "cos_Ac_negg": [0.6015437841415405, 0.6266802549362183, 0.6626092195510864], "r_norm": [0.17999890446662903, 0.1781638264656067, 0.17986194789409637]}, {"step": 6500, "epoch": 13, "train_loss": 0.0003943365009035915, "cos_r_negg": [0.2314969003200531, 0.26934683322906494, 0.30806347727775574], "cos_innovation_negg": [0.2314969003200531, 0.26934683322906494, 0.30806347727775574], "cos_apical_negg": [-0.006277681328356266, 0.04623973369598389, 0.08112269639968872], "cos_Ac_negg": [0.6035950183868408, 0.6283788681030273, 0.663296103477478], "r_norm": [0.18169091641902924, 0.1796780228614807, 0.18181093037128448], "ldr": {"dL_sdil": -0.009350590407848358, "dL_bp": -0.019655853509902954, "ratio": 0.4757153080703095}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9724, "test_loss": 0.14104432067871095}, {"step": 6600, "epoch": 14, "train_loss": 0.00024284770188387483, "cos_r_negg": [0.22123859822750092, 0.2598249316215515, 0.29724153876304626], "cos_innovation_negg": [0.22123859822750092, 0.2598249316215515, 0.29724153876304626], "cos_apical_negg": [-0.008594653569161892, 0.045161910355091095, 0.07877635955810547], "cos_Ac_negg": [0.5981976985931396, 0.6258737444877625, 0.662596583366394], "r_norm": [0.18268665671348572, 0.1807129681110382, 0.1825793981552124]}, {"step": 6700, "epoch": 14, "train_loss": 0.000819356762804091, "cos_r_negg": [0.21892967820167542, 0.25876832008361816, 0.2971835136413574], "cos_innovation_negg": [0.21892967820167542, 0.25876832008361816, 0.2971835136413574], "cos_apical_negg": [-0.008528337813913822, 0.04554855823516846, 0.07917456328868866], "cos_Ac_negg": [0.5992168188095093, 0.6250213384628296, 0.6616549491882324], "r_norm": [0.18037615716457367, 0.17839638888835907, 0.1804060935974121]}, {"step": 6800, "epoch": 14, "train_loss": 0.0006911448435857892, "cos_r_negg": [0.21183283627033234, 0.2506535053253174, 0.2896501421928406], "cos_innovation_negg": [0.21183283627033234, 0.2506535053253174, 0.2896501421928406], "cos_apical_negg": [-0.008370094001293182, 0.04392202943563461, 0.07889318466186523], "cos_Ac_negg": [0.5985738039016724, 0.6242616176605225, 0.6628355979919434], "r_norm": [0.17608055472373962, 0.17446398735046387, 0.17615440487861633]}, {"step": 6900, "epoch": 14, "train_loss": 0.0008076176745817065, "cos_r_negg": [0.21059472858905792, 0.24934405088424683, 0.28680405020713806], "cos_innovation_negg": [0.21059472858905792, 0.24934405088424683, 0.28680405020713806], "cos_apical_negg": [-0.007636239752173424, 0.043772052973508835, 0.07701065391302109], "cos_Ac_negg": [0.5949746966362, 0.6231884956359863, 0.6632137298583984], "r_norm": [0.1791018843650818, 0.1774611920118332, 0.17879526317119598]}, {"step": 7000, "epoch": 14, "train_loss": 0.0006948065129108727, "cos_r_negg": [0.2116452157497406, 0.250020295381546, 0.28750133514404297], "cos_innovation_negg": [0.2116452157497406, 0.250020295381546, 0.28750133514404297], "cos_apical_negg": [-0.007414512801915407, 0.04454676806926727, 0.07724614441394806], "cos_Ac_negg": [0.5964549779891968, 0.6238608360290527, 0.662742555141449], "r_norm": [0.18482258915901184, 0.18285025656223297, 0.1847810596227646], "ldr": {"dL_sdil": -0.009862080216407776, "dL_bp": -0.021312817931175232, "ratio": 0.46272999883239563}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9728, "test_loss": 0.1435254264831543}], "final": {"test_acc": 0.9728, "test_loss": 0.1435254264831543, "wall_s": 23.386398792266846, "cos_r_negg": [0.21129368245601654, 0.24853135645389557, 0.28675156831741333], "cos_innovation_negg": [0.21129368245601654, 0.24853135645389557, 0.28675156831741333], "cos_apical_negg": [-0.006689662579447031, 0.044270001351833344, 0.077663853764534], "cos_Ac_negg": [0.5961201190948486, 0.6233915090560913, 0.6630139350891113]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_residual_s2.json b/results/nuis_ctrl_v1_rho0p2_residual_s2.json new file mode 100644 index 0000000..18d8f5a --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_residual_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_residual_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.01767529547214508, 0.03728756308555603, 0.002514312509447336], "cos_innovation_negg": [0.01767529547214508, 0.03728756308555603, 0.002514312509447336], "cos_apical_negg": [0.015415688045322895, 0.03417167812585831, 0.0006933959666639566], "cos_Ac_negg": [0.01767658442258835, 0.03728780150413513, 0.002514585852622986], "r_norm": [4.749824523925781, 4.774453163146973, 4.856010913848877], "ldr": {"dL_sdil": -0.041074514389038086, "dL_bp": -0.24915242195129395, "ratio": 0.1648569741660694}}, {"step": 100, "epoch": 0, "train_loss": 0.17509281635284424, "cos_r_negg": [0.4975433349609375, 0.40333205461502075, 0.3688212037086487], "cos_innovation_negg": [0.4975433349609375, 0.40333205461502075, 0.3688212037086487], "cos_apical_negg": [0.11618749052286148, 0.11813625693321228, 0.16940230131149292], "cos_Ac_negg": [0.49778246879577637, 0.4039401412010193, 0.3688860535621643], "r_norm": [0.8709042072296143, 0.8829477429389954, 0.8956848382949829]}, {"step": 200, "epoch": 0, "train_loss": 0.32225701212882996, "cos_r_negg": [0.49504613876342773, 0.44531792402267456, 0.40527746081352234], "cos_innovation_negg": [0.49504613876342773, 0.44531792402267456, 0.40527746081352234], "cos_apical_negg": [0.12389877438545227, 0.1307678371667862, 0.16186591982841492], "cos_Ac_negg": [0.4941730499267578, 0.4442378878593445, 0.40495961904525757], "r_norm": [0.714240312576294, 0.7208932042121887, 0.737350583076477]}, {"step": 300, "epoch": 0, "train_loss": 0.19723594188690186, "cos_r_negg": [0.5261498689651489, 0.46736329793930054, 0.42248106002807617], "cos_innovation_negg": [0.5261498689651489, 0.46736329793930054, 0.42248106002807617], "cos_apical_negg": [0.08763691782951355, 0.10477475821971893, 0.14658178389072418], "cos_Ac_negg": [0.524702787399292, 0.46649879217147827, 0.4217778444290161], "r_norm": [0.5128189325332642, 0.5183968544006348, 0.5274415016174316]}, {"step": 400, "epoch": 0, "train_loss": 0.15750786662101746, "cos_r_negg": [0.5310730934143066, 0.5144582390785217, 0.4407214820384979], "cos_innovation_negg": [0.5310730934143066, 0.5144582390785217, 0.4407214820384979], "cos_apical_negg": [0.08404471725225449, 0.11787910014390945, 0.14171254634857178], "cos_Ac_negg": [0.5304864645004272, 0.5137860774993896, 0.4403280019760132], "r_norm": [0.508476734161377, 0.5139731168746948, 0.5231341123580933]}, {"epoch_end": 0, "step": 469, "test_acc": 0.947, "test_loss": 0.17562377853393554}, {"step": 500, "epoch": 1, "train_loss": 0.1597687005996704, "cos_r_negg": [0.5122919678688049, 0.5138441324234009, 0.4631630480289459], "cos_innovation_negg": [0.5122919678688049, 0.5138441324234009, 0.4631630480289459], "cos_apical_negg": [0.08693978935480118, 0.11116659641265869, 0.15668684244155884], "cos_Ac_negg": [0.5122904181480408, 0.5142987966537476, 0.46278059482574463], "r_norm": [0.4930664896965027, 0.4995163679122925, 0.5074531435966492], "ldr": {"dL_sdil": -0.015003561973571777, "dL_bp": -0.012398585677146912, "ratio": 1.2101026975380234}}, {"step": 600, "epoch": 1, "train_loss": 0.09708543121814728, "cos_r_negg": [0.5431234240531921, 0.531376838684082, 0.44608527421951294], "cos_innovation_negg": [0.5431234240531921, 0.531376838684082, 0.44608527421951294], "cos_apical_negg": [0.08340280503034592, 0.10146048665046692, 0.13465362787246704], "cos_Ac_negg": [0.5429235696792603, 0.5293044447898865, 0.445862352848053], "r_norm": [0.4237655699253082, 0.42812156677246094, 0.4349307417869568]}, {"step": 700, "epoch": 1, "train_loss": 0.1818649172782898, "cos_r_negg": [0.5616540312767029, 0.5386641025543213, 0.4616681933403015], "cos_innovation_negg": [0.5616540312767029, 0.5386641025543213, 0.4616681933403015], "cos_apical_negg": [0.08378094434738159, 0.11076634377241135, 0.13736295700073242], "cos_Ac_negg": [0.5616635680198669, 0.537530779838562, 0.4620056748390198], "r_norm": [0.42998117208480835, 0.4348870813846588, 0.4410648047924042]}, {"step": 800, "epoch": 1, "train_loss": 0.1649283766746521, "cos_r_negg": [0.557459831237793, 0.553570568561554, 0.4833041727542877], "cos_innovation_negg": [0.557459831237793, 0.553570568561554, 0.4833041727542877], "cos_apical_negg": [0.07074064016342163, 0.09915652871131897, 0.15585289895534515], "cos_Ac_negg": [0.5573357343673706, 0.5529607534408569, 0.4829045534133911], "r_norm": [0.351260244846344, 0.35430845618247986, 0.3613123893737793]}, {"step": 900, "epoch": 1, "train_loss": 0.18128633499145508, "cos_r_negg": [0.5602138042449951, 0.5554276704788208, 0.4848535656929016], "cos_innovation_negg": [0.5602138042449951, 0.5554276704788208, 0.4848535656929016], "cos_apical_negg": [0.07547470927238464, 0.10947113484144211, 0.1396656185388565], "cos_Ac_negg": [0.5599839091300964, 0.5538674592971802, 0.4847639799118042], "r_norm": [0.41619181632995605, 0.4217885434627533, 0.4277922213077545]}, {"epoch_end": 1, "step": 938, "test_acc": 0.96, "test_loss": 0.13628402557373046}, {"step": 1000, "epoch": 2, "train_loss": 0.014245163649320602, "cos_r_negg": [0.5582895278930664, 0.5609408020973206, 0.48939502239227295], "cos_innovation_negg": [0.5582895278930664, 0.5609408020973206, 0.48939502239227295], "cos_apical_negg": [0.03537789359688759, 0.09007109701633453, 0.12155909091234207], "cos_Ac_negg": [0.558417797088623, 0.5603756904602051, 0.48841458559036255], "r_norm": [0.2976021468639374, 0.2996687889099121, 0.30765464901924133], "ldr": {"dL_sdil": -0.0068322643637657166, "dL_bp": -0.005730554461479187, "ratio": 1.1922518858676292}}, {"step": 1100, "epoch": 2, "train_loss": 0.10803592205047607, "cos_r_negg": [0.5534619092941284, 0.5789526700973511, 0.5089566707611084], "cos_innovation_negg": [0.5534619092941284, 0.5789526700973511, 0.5089566707611084], "cos_apical_negg": [0.0462685264647007, 0.10204433649778366, 0.13513657450675964], "cos_Ac_negg": [0.5526604652404785, 0.5777796506881714, 0.509196400642395], "r_norm": [0.38753318786621094, 0.39252525568008423, 0.40004366636276245]}, {"step": 1200, "epoch": 2, "train_loss": 0.19375663995742798, "cos_r_negg": [0.5653595924377441, 0.5850834250450134, 0.5095187425613403], "cos_innovation_negg": [0.5653595924377441, 0.5850834250450134, 0.5095187425613403], "cos_apical_negg": [0.057870879769325256, 0.09330567717552185, 0.13008034229278564], "cos_Ac_negg": [0.5664764046669006, 0.5854951739311218, 0.5086528062820435], "r_norm": [0.3420349359512329, 0.34734806418418884, 0.35258787870407104]}, {"step": 1300, "epoch": 2, "train_loss": 0.08626707643270493, "cos_r_negg": [0.5832798480987549, 0.5954384803771973, 0.5044431686401367], "cos_innovation_negg": [0.5832798480987549, 0.5954384803771973, 0.5044431686401367], "cos_apical_negg": [0.050631701946258545, 0.0940709263086319, 0.13271114230155945], "cos_Ac_negg": [0.5850399136543274, 0.5957673788070679, 0.5031443238258362], "r_norm": [0.33668726682662964, 0.3421662747859955, 0.3458006680011749]}, {"step": 1400, "epoch": 2, "train_loss": 0.11529648303985596, "cos_r_negg": [0.573486328125, 0.5938014984130859, 0.5095750093460083], "cos_innovation_negg": [0.573486328125, 0.5938014984130859, 0.5095750093460083], "cos_apical_negg": [0.04238450899720192, 0.08704474568367004, 0.11725302040576935], "cos_Ac_negg": [0.5743416547775269, 0.5939078330993652, 0.5077441930770874], "r_norm": [0.2862524390220642, 0.28860414028167725, 0.29272639751434326]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9655, "test_loss": 0.11371440372467041}, {"step": 1500, "epoch": 3, "train_loss": 0.08936333656311035, "cos_r_negg": [0.5783503651618958, 0.5958328247070312, 0.5227053165435791], "cos_innovation_negg": [0.5783503651618958, 0.5958328247070312, 0.5227053165435791], "cos_apical_negg": [0.058128561824560165, 0.08985227346420288, 0.12818922102451324], "cos_Ac_negg": [0.5810917019844055, 0.5977128744125366, 0.5219247341156006], "r_norm": [0.30634763836860657, 0.3092188835144043, 0.3137034773826599], "ldr": {"dL_sdil": -0.009206898510456085, "dL_bp": -0.008490093052387238, "ratio": 1.0844284572201828}}, {"step": 1600, "epoch": 3, "train_loss": 0.03847982734441757, "cos_r_negg": [0.5721505880355835, 0.6015081405639648, 0.538394570350647], "cos_innovation_negg": [0.5721505880355835, 0.6015081405639648, 0.538394570350647], "cos_apical_negg": [0.047964029014110565, 0.08151105046272278, 0.12632524967193604], "cos_Ac_negg": [0.5769691467285156, 0.603405773639679, 0.537426233291626], "r_norm": [0.2822626829147339, 0.28604504466056824, 0.28683629631996155]}, {"step": 1700, "epoch": 3, "train_loss": 0.032343361526727676, "cos_r_negg": [0.5575027465820312, 0.5930342674255371, 0.5395357608795166], "cos_innovation_negg": [0.5575027465820312, 0.5930342674255371, 0.5395357608795166], "cos_apical_negg": [0.05143904685974121, 0.08221805840730667, 0.133917436003685], "cos_Ac_negg": [0.5824142694473267, 0.6069841384887695, 0.5476691722869873], "r_norm": [0.27640581130981445, 0.28019270300865173, 0.28369253873825073]}, {"step": 1800, "epoch": 3, "train_loss": 0.026220418512821198, "cos_r_negg": [0.5699102878570557, 0.6016329526901245, 0.5518378019332886], "cos_innovation_negg": [0.5699102878570557, 0.6016329526901245, 0.5518378019332886], "cos_apical_negg": [0.03474421799182892, 0.07611443102359772, 0.11839596927165985], "cos_Ac_negg": [0.5838421583175659, 0.6098796129226685, 0.5545596480369568], "r_norm": [0.21730363368988037, 0.22058764100074768, 0.22224876284599304]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9667, "test_loss": 0.11673946208953857}, {"step": 1900, "epoch": 4, "train_loss": 0.02581382356584072, "cos_r_negg": [0.5651746988296509, 0.6027153730392456, 0.5560572147369385], "cos_innovation_negg": [0.5651746988296509, 0.6027153730392456, 0.5560572147369385], "cos_apical_negg": [0.03459694981575012, 0.07795149087905884, 0.11487835645675659], "cos_Ac_negg": [0.5787738561630249, 0.613370418548584, 0.5600499510765076], "r_norm": [0.24008040130138397, 0.24260438978672028, 0.24710731208324432]}, {"step": 2000, "epoch": 4, "train_loss": 0.043413709849119186, "cos_r_negg": [0.5390135049819946, 0.5880950689315796, 0.5479379892349243], "cos_innovation_negg": [0.5390135049819946, 0.5880950689315796, 0.5479379892349243], "cos_apical_negg": [0.033422425389289856, 0.08546341955661774, 0.12646059691905975], "cos_Ac_negg": [0.571420431137085, 0.6140965819358826, 0.5627903342247009], "r_norm": [0.23989640176296234, 0.24191661179065704, 0.24486878514289856], "ldr": {"dL_sdil": -0.00921265035867691, "dL_bp": -0.009466208517551422, "ratio": 0.9732143911256143}}, {"step": 2100, "epoch": 4, "train_loss": 0.017345048487186432, "cos_r_negg": [0.5627932548522949, 0.5979176759719849, 0.5574103593826294], "cos_innovation_negg": [0.5627932548522949, 0.5979176759719849, 0.5574103593826294], "cos_apical_negg": [0.02551228180527687, 0.0805293619632721, 0.11841922998428345], "cos_Ac_negg": [0.5939663648605347, 0.6210989952087402, 0.5725558996200562], "r_norm": [0.2211616337299347, 0.22288961708545685, 0.22653460502624512]}, {"step": 2200, "epoch": 4, "train_loss": 0.005916507449001074, "cos_r_negg": [0.5655982494354248, 0.6089769005775452, 0.5700035095214844], "cos_innovation_negg": [0.5655982494354248, 0.6089769005775452, 0.5700035095214844], "cos_apical_negg": [0.036312371492385864, 0.08154720813035965, 0.1202714741230011], "cos_Ac_negg": [0.5922354459762573, 0.6318511962890625, 0.5815303921699524], "r_norm": [0.24052634835243225, 0.24335400760173798, 0.24619004130363464]}, {"step": 2300, "epoch": 4, "train_loss": 0.16136693954467773, "cos_r_negg": [0.5516446828842163, 0.5938196182250977, 0.559701681137085], "cos_innovation_negg": [0.5516446828842163, 0.5938196182250977, 0.559701681137085], "cos_apical_negg": [0.032704997807741165, 0.07346057146787643, 0.11832443624734879], "cos_Ac_negg": [0.5918793082237244, 0.6262285709381104, 0.5788072347640991], "r_norm": [0.1954016089439392, 0.19766303896903992, 0.1996578574180603]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9719, "test_loss": 0.10524423446655273}, {"step": 2400, "epoch": 5, "train_loss": 0.0710926428437233, "cos_r_negg": [0.5653858184814453, 0.6059912443161011, 0.5625099539756775], "cos_innovation_negg": [0.5653858184814453, 0.6059912443161011, 0.5625099539756775], "cos_apical_negg": [0.016023822128772736, 0.06713022291660309, 0.10897444188594818], "cos_Ac_negg": [0.6120545268058777, 0.6424462795257568, 0.5879056453704834], "r_norm": [0.18106362223625183, 0.18413177132606506, 0.18646585941314697]}, {"step": 2500, "epoch": 5, "train_loss": 0.01808163896203041, "cos_r_negg": [0.5108247995376587, 0.5604627132415771, 0.5431246161460876], "cos_innovation_negg": [0.5108247995376587, 0.5604627132415771, 0.5431246161460876], "cos_apical_negg": [0.016557615250349045, 0.07462646067142487, 0.11858949065208435], "cos_Ac_negg": [0.5983515977859497, 0.6272869110107422, 0.5899462103843689], "r_norm": [0.26577088236808777, 0.269166499376297, 0.270310640335083], "ldr": {"dL_sdil": -0.012525133788585663, "dL_bp": -0.013445675373077393, "ratio": 0.9315362331047384}}, {"step": 2600, "epoch": 5, "train_loss": 0.02552671916782856, "cos_r_negg": [0.5264327526092529, 0.5699678659439087, 0.5497574806213379], "cos_innovation_negg": [0.5264327526092529, 0.5699678659439087, 0.5497574806213379], "cos_apical_negg": [0.03641090542078018, 0.08105245977640152, 0.12503226101398468], "cos_Ac_negg": [0.6009337306022644, 0.636478066444397, 0.589719831943512], "r_norm": [0.24126897752285004, 0.24377411603927612, 0.24600544571876526]}, {"step": 2700, "epoch": 5, "train_loss": 0.019685301929712296, "cos_r_negg": [0.540636420249939, 0.578506588935852, 0.5580445528030396], "cos_innovation_negg": [0.540636420249939, 0.578506588935852, 0.5580445528030396], "cos_apical_negg": [0.0297347754240036, 0.06568491458892822, 0.11520075798034668], "cos_Ac_negg": [0.6147531270980835, 0.6533650755882263, 0.6031450629234314], "r_norm": [0.19449009001255035, 0.19642406702041626, 0.19922471046447754]}, {"step": 2800, "epoch": 5, "train_loss": 0.07865852862596512, "cos_r_negg": [0.5507789850234985, 0.581382155418396, 0.5632185935974121], "cos_innovation_negg": [0.5507789850234985, 0.581382155418396, 0.5632185935974121], "cos_apical_negg": [0.03421325236558914, 0.07255618274211884, 0.1184140294790268], "cos_Ac_negg": [0.6072455048561096, 0.6386983394622803, 0.5979628562927246], "r_norm": [0.21219971776008606, 0.21393488347530365, 0.21595433354377747]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9715, "test_loss": 0.10897763557434081}, {"step": 2900, "epoch": 6, "train_loss": 0.009145213291049004, "cos_r_negg": [0.5207064747810364, 0.5689835548400879, 0.5601541996002197], "cos_innovation_negg": [0.5207064747810364, 0.5689835548400879, 0.5601541996002197], "cos_apical_negg": [0.015072949230670929, 0.0640232041478157, 0.11946884542703629], "cos_Ac_negg": [0.60323166847229, 0.6468353271484375, 0.6125844717025757], "r_norm": [0.20500819385051727, 0.20745383203029633, 0.20849820971488953]}, {"step": 3000, "epoch": 6, "train_loss": 0.0715070441365242, "cos_r_negg": [0.49930766224861145, 0.5488592386245728, 0.5531019568443298], "cos_innovation_negg": [0.49930766224861145, 0.5488592386245728, 0.5531019568443298], "cos_apical_negg": [0.013733266852796078, 0.06253304332494736, 0.11913203448057175], "cos_Ac_negg": [0.6024250984191895, 0.6468963623046875, 0.6145122647285461], "r_norm": [0.19223961234092712, 0.19386914372444153, 0.19674967229366302], "ldr": {"dL_sdil": -0.006430089473724365, "dL_bp": -0.007251575589179993, "ratio": 0.886716189419392}}, {"step": 3100, "epoch": 6, "train_loss": 0.01816060021519661, "cos_r_negg": [0.48341017961502075, 0.5377269983291626, 0.5396313071250916], "cos_innovation_negg": [0.48341017961502075, 0.5377269983291626, 0.5396313071250916], "cos_apical_negg": [0.02214656211435795, 0.06700734794139862, 0.1243247240781784], "cos_Ac_negg": [0.6071529388427734, 0.6570723056793213, 0.6200355291366577], "r_norm": [0.21362580358982086, 0.21481359004974365, 0.2180177867412567]}, {"step": 3200, "epoch": 6, "train_loss": 0.024276409298181534, "cos_r_negg": [0.4661824405193329, 0.5266361236572266, 0.5287977457046509], "cos_innovation_negg": [0.4661824405193329, 0.5266361236572266, 0.5287977457046509], "cos_apical_negg": [0.01684332638978958, 0.0681089237332344, 0.12278212606906891], "cos_Ac_negg": [0.6078313589096069, 0.658248782157898, 0.6226560473442078], "r_norm": [0.21652251482009888, 0.2184813767671585, 0.21925285458564758]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9733, "test_loss": 0.1132070237159729}, {"step": 3300, "epoch": 7, "train_loss": 0.011985664255917072, "cos_r_negg": [0.4865093231201172, 0.5362573862075806, 0.5370480418205261], "cos_innovation_negg": [0.4865093231201172, 0.5362573862075806, 0.5370480418205261], "cos_apical_negg": [0.00796530768275261, 0.0603739395737648, 0.12149442732334137], "cos_Ac_negg": [0.6164408922195435, 0.6616761088371277, 0.6272478103637695], "r_norm": [0.17885759472846985, 0.18030945956707, 0.182717964053154]}, {"step": 3400, "epoch": 7, "train_loss": 0.03560802340507507, "cos_r_negg": [0.44791898131370544, 0.5046046376228333, 0.51435387134552], "cos_innovation_negg": [0.44791898131370544, 0.5046046376228333, 0.51435387134552], "cos_apical_negg": [0.0138439005240798, 0.0664428249001503, 0.12263033539056778], "cos_Ac_negg": [0.6202693581581116, 0.6579979658126831, 0.6237648725509644], "r_norm": [0.19945357739925385, 0.20128455758094788, 0.20433785021305084]}, {"step": 3500, "epoch": 7, "train_loss": 0.01580345816910267, "cos_r_negg": [0.421935498714447, 0.4741741716861725, 0.497981458902359], "cos_innovation_negg": [0.421935498714447, 0.4741741716861725, 0.497981458902359], "cos_apical_negg": [0.021678755059838295, 0.062351614236831665, 0.1227707713842392], "cos_Ac_negg": [0.6099575757980347, 0.6519378423690796, 0.629919171333313], "r_norm": [0.16567443311214447, 0.1672157198190689, 0.16966548562049866], "ldr": {"dL_sdil": -0.006317123770713806, "dL_bp": -0.008358336985111237, "ratio": 0.7557871598102042}}, {"step": 3600, "epoch": 7, "train_loss": 0.0018474484095349908, "cos_r_negg": [0.4476993978023529, 0.494841992855072, 0.513182520866394], "cos_innovation_negg": [0.4476993978023529, 0.494841992855072, 0.513182520866394], "cos_apical_negg": [0.018582699820399284, 0.06624887883663177, 0.12038591504096985], "cos_Ac_negg": [0.6241501569747925, 0.6635108590126038, 0.6401136517524719], "r_norm": [0.18853527307510376, 0.19027414917945862, 0.19222232699394226]}, {"step": 3700, "epoch": 7, "train_loss": 0.009599377401173115, "cos_r_negg": [0.40976840257644653, 0.46963703632354736, 0.4920122027397156], "cos_innovation_negg": [0.40976840257644653, 0.46963703632354736, 0.4920122027397156], "cos_apical_negg": [0.012469684705138206, 0.062279049307107925, 0.11710610240697861], "cos_Ac_negg": [0.6172502040863037, 0.6636425256729126, 0.6380182504653931], "r_norm": [0.17751789093017578, 0.17908617854118347, 0.17978350818157196]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9724, "test_loss": 0.11808997554779052}, {"step": 3800, "epoch": 8, "train_loss": 0.004688822664320469, "cos_r_negg": [0.3743799924850464, 0.43885743618011475, 0.47301656007766724], "cos_innovation_negg": [0.3743799924850464, 0.43885743618011475, 0.47301656007766724], "cos_apical_negg": [0.0036443760618567467, 0.06525722146034241, 0.12600061297416687], "cos_Ac_negg": [0.6125617027282715, 0.6577926278114319, 0.6411197185516357], "r_norm": [0.20205265283584595, 0.203693687915802, 0.2046319544315338]}, {"step": 3900, "epoch": 8, "train_loss": 0.006033105310052633, "cos_r_negg": [0.35067611932754517, 0.4182766079902649, 0.4520740807056427], "cos_innovation_negg": [0.35067611932754517, 0.4182766079902649, 0.4520740807056427], "cos_apical_negg": [-0.0008125752210617065, 0.06210014969110489, 0.11539337784051895], "cos_Ac_negg": [0.6148279905319214, 0.6582210659980774, 0.6479083895683289], "r_norm": [0.16353049874305725, 0.16419976949691772, 0.16604915261268616]}, {"step": 4000, "epoch": 8, "train_loss": 0.004340995103120804, "cos_r_negg": [0.3396536111831665, 0.4068759083747864, 0.4441215991973877], "cos_innovation_negg": [0.3396536111831665, 0.4068759083747864, 0.4441215991973877], "cos_apical_negg": [0.0033229896798729897, 0.06548186391592026, 0.11455140262842178], "cos_Ac_negg": [0.6149129867553711, 0.6597929000854492, 0.6462996602058411], "r_norm": [0.18419606983661652, 0.1867668479681015, 0.18584467470645905], "ldr": {"dL_sdil": -0.009168088436126709, "dL_bp": -0.01314333826303482, "ratio": 0.6975464111664567}}, {"step": 4100, "epoch": 8, "train_loss": 0.00870001781731844, "cos_r_negg": [0.36556869745254517, 0.42563098669052124, 0.4615456461906433], "cos_innovation_negg": [0.36556869745254517, 0.42563098669052124, 0.4615456461906433], "cos_apical_negg": [0.011623446829617023, 0.059345588088035583, 0.11349816620349884], "cos_Ac_negg": [0.6119230389595032, 0.6618313789367676, 0.650639533996582], "r_norm": [0.17602267861366272, 0.17788194119930267, 0.1778104603290558]}, {"step": 4200, "epoch": 8, "train_loss": 0.014143695123493671, "cos_r_negg": [0.36391937732696533, 0.41778942942619324, 0.4546824097633362], "cos_innovation_negg": [0.36391937732696533, 0.41778942942619324, 0.4546824097633362], "cos_apical_negg": [0.017321418970823288, 0.06580847501754761, 0.11721920967102051], "cos_Ac_negg": [0.6158136129379272, 0.662009596824646, 0.6472315788269043], "r_norm": [0.19318892061710358, 0.19453930854797363, 0.19671779870986938]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9732, "test_loss": 0.12282343168258666}, {"step": 4300, "epoch": 9, "train_loss": 0.0032092463225126266, "cos_r_negg": [0.3330169916152954, 0.3981086313724518, 0.4332948327064514], "cos_innovation_negg": [0.3330169916152954, 0.3981086313724518, 0.4332948327064514], "cos_apical_negg": [0.008161344565451145, 0.06398330628871918, 0.11448262631893158], "cos_Ac_negg": [0.6102118492126465, 0.6607881784439087, 0.648625910282135], "r_norm": [0.19351688027381897, 0.1955786794424057, 0.1956542432308197]}, {"step": 4400, "epoch": 9, "train_loss": 0.001069177407771349, "cos_r_negg": [0.31436893343925476, 0.3792405128479004, 0.4175804853439331], "cos_innovation_negg": [0.31436893343925476, 0.3792405128479004, 0.4175804853439331], "cos_apical_negg": [0.0034335567615926266, 0.060172080993652344, 0.11194276809692383], "cos_Ac_negg": [0.6074249148368835, 0.657621443271637, 0.6497157216072083], "r_norm": [0.17512217164039612, 0.17642568051815033, 0.17667174339294434]}, {"step": 4500, "epoch": 9, "train_loss": 0.08040723204612732, "cos_r_negg": [0.30991387367248535, 0.36579954624176025, 0.40476682782173157], "cos_innovation_negg": [0.30991387367248535, 0.36579954624176025, 0.40476682782173157], "cos_apical_negg": [0.012391677126288414, 0.06332561373710632, 0.11174505949020386], "cos_Ac_negg": [0.605425238609314, 0.6566510200500488, 0.6502143740653992], "r_norm": [0.18233337998390198, 0.1843034327030182, 0.1852201372385025], "ldr": {"dL_sdil": -0.00801856815814972, "dL_bp": -0.01233837753534317, "ratio": 0.6498883775586055}}, {"step": 4600, "epoch": 9, "train_loss": 0.003453320823609829, "cos_r_negg": [0.31546252965927124, 0.3722880482673645, 0.4147198796272278], "cos_innovation_negg": [0.31546252965927124, 0.3722880482673645, 0.4147198796272278], "cos_apical_negg": [0.012299302965402603, 0.05744505301117897, 0.11137974262237549], "cos_Ac_negg": [0.6120774149894714, 0.6598763465881348, 0.6571483016014099], "r_norm": [0.17425686120986938, 0.1754762828350067, 0.17738035321235657]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9724, "test_loss": 0.12562893190383911}, {"step": 4700, "epoch": 10, "train_loss": 0.001479143276810646, "cos_r_negg": [0.2941581904888153, 0.3462350368499756, 0.38551393151283264], "cos_innovation_negg": [0.2941581904888153, 0.3462350368499756, 0.38551393151283264], "cos_apical_negg": [0.006535946391522884, 0.055504683405160904, 0.1015453189611435], "cos_Ac_negg": [0.6063218116760254, 0.6541507244110107, 0.6517965793609619], "r_norm": [0.17782631516456604, 0.17889955639839172, 0.18163086473941803]}, {"step": 4800, "epoch": 10, "train_loss": 0.005755386780947447, "cos_r_negg": [0.28948140144348145, 0.3457110524177551, 0.3862270712852478], "cos_innovation_negg": [0.28948140144348145, 0.3457110524177551, 0.3862270712852478], "cos_apical_negg": [0.0035658692941069603, 0.06003706157207489, 0.10304266214370728], "cos_Ac_negg": [0.6079223155975342, 0.6549617052078247, 0.6528621315956116], "r_norm": [0.16757214069366455, 0.1692620813846588, 0.17122630774974823]}, {"step": 4900, "epoch": 10, "train_loss": 0.0012413340155035257, "cos_r_negg": [0.28225111961364746, 0.33978506922721863, 0.37782710790634155], "cos_innovation_negg": [0.28225111961364746, 0.33978506922721863, 0.37782710790634155], "cos_apical_negg": [4.943110980093479e-05, 0.05673644691705704, 0.10129523277282715], "cos_Ac_negg": [0.6072084307670593, 0.6557126045227051, 0.6524120569229126], "r_norm": [0.16640053689479828, 0.1678115725517273, 0.17007088661193848]}, {"step": 5000, "epoch": 10, "train_loss": 0.0007068759296089411, "cos_r_negg": [0.27025720477104187, 0.3285697102546692, 0.37197166681289673], "cos_innovation_negg": [0.27025720477104187, 0.3285697102546692, 0.37197166681289673], "cos_apical_negg": [0.0031626210547983646, 0.0593554750084877, 0.1041841208934784], "cos_Ac_negg": [0.605614185333252, 0.6529886722564697, 0.6520781517028809], "r_norm": [0.16406592726707458, 0.1654266119003296, 0.16743773221969604], "ldr": {"dL_sdil": -0.0072349607944488525, "dL_bp": -0.011477462947368622, "ratio": 0.630362374300461}}, {"step": 5100, "epoch": 10, "train_loss": 0.0020517022348940372, "cos_r_negg": [0.2734268009662628, 0.3238193392753601, 0.3673960566520691], "cos_innovation_negg": [0.2734268009662628, 0.3238193392753601, 0.3673960566520691], "cos_apical_negg": [0.0040702782571315765, 0.05675593391060829, 0.10166744887828827], "cos_Ac_negg": [0.6023961305618286, 0.6538525223731995, 0.6518510580062866], "r_norm": [0.1594313681125641, 0.16045798361301422, 0.1636919528245926]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9737, "test_loss": 0.12942691631317138}, {"step": 5200, "epoch": 11, "train_loss": 0.0009447101037949324, "cos_r_negg": [0.27504998445510864, 0.32590317726135254, 0.37007665634155273], "cos_innovation_negg": [0.27504998445510864, 0.32590317726135254, 0.37007665634155273], "cos_apical_negg": [0.005906685255467892, 0.05529320240020752, 0.10000424087047577], "cos_Ac_negg": [0.5991940498352051, 0.6549771428108215, 0.6554460525512695], "r_norm": [0.1676741987466812, 0.16874822974205017, 0.17141152918338776]}, {"step": 5300, "epoch": 11, "train_loss": 0.002786048222333193, "cos_r_negg": [0.25757014751434326, 0.31285297870635986, 0.3559439182281494], "cos_innovation_negg": [0.25757014751434326, 0.31285297870635986, 0.3559439182281494], "cos_apical_negg": [0.0007860306650400162, 0.05470998212695122, 0.09603939950466156], "cos_Ac_negg": [0.5953935384750366, 0.6516157984733582, 0.6553336977958679], "r_norm": [0.16268977522850037, 0.1643703281879425, 0.1648414582014084]}, {"step": 5400, "epoch": 11, "train_loss": 0.0017298487946391106, "cos_r_negg": [0.25706911087036133, 0.30632898211479187, 0.3487485647201538], "cos_innovation_negg": [0.25706911087036133, 0.30632898211479187, 0.3487485647201538], "cos_apical_negg": [0.0017285854555666447, 0.05273089557886124, 0.09379982948303223], "cos_Ac_negg": [0.5986457467079163, 0.6504547595977783, 0.6536934971809387], "r_norm": [0.15570726990699768, 0.15690907835960388, 0.15948693454265594]}, {"step": 5500, "epoch": 11, "train_loss": 0.0011652226094156504, "cos_r_negg": [0.24918265640735626, 0.29930007457733154, 0.34484216570854187], "cos_innovation_negg": [0.24918265640735626, 0.29930007457733154, 0.34484216570854187], "cos_apical_negg": [0.0025983727537095547, 0.05365142226219177, 0.09881532192230225], "cos_Ac_negg": [0.5980652570724487, 0.6493713855743408, 0.6545578837394714], "r_norm": [0.16840405762195587, 0.16959857940673828, 0.1719251573085785], "ldr": {"dL_sdil": -0.008044131100177765, "dL_bp": -0.013246871531009674, "ratio": 0.607247611735889}}, {"step": 5600, "epoch": 11, "train_loss": 0.00058410270139575, "cos_r_negg": [0.23608864843845367, 0.29058295488357544, 0.3340868055820465], "cos_innovation_negg": [0.23608864843845367, 0.29058295488357544, 0.3340868055820465], "cos_apical_negg": [-0.0014969015028327703, 0.052514299750328064, 0.0947546735405922], "cos_Ac_negg": [0.5975660085678101, 0.6485641002655029, 0.6523817181587219], "r_norm": [0.15986914932727814, 0.16145658493041992, 0.16321061551570892]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9732, "test_loss": 0.132458203125}, {"step": 5700, "epoch": 12, "train_loss": 0.0013621447142213583, "cos_r_negg": [0.24343115091323853, 0.29329371452331543, 0.3370521664619446], "cos_innovation_negg": [0.24343115091323853, 0.29329371452331543, 0.3370521664619446], "cos_apical_negg": [0.00218449835665524, 0.05301017314195633, 0.0952487364411354], "cos_Ac_negg": [0.5987480878829956, 0.6501702070236206, 0.6578517556190491], "r_norm": [0.16959352791309357, 0.17127740383148193, 0.17256619036197662]}, {"step": 5800, "epoch": 12, "train_loss": 0.0005102292634546757, "cos_r_negg": [0.23084500432014465, 0.28513145446777344, 0.3270755410194397], "cos_innovation_negg": [0.23084500432014465, 0.28513145446777344, 0.3270755410194397], "cos_apical_negg": [-0.0009406257886439562, 0.053193897008895874, 0.09250657260417938], "cos_Ac_negg": [0.5968866348266602, 0.6491766571998596, 0.656719446182251], "r_norm": [0.16447961330413818, 0.1661590039730072, 0.1673867404460907]}, {"step": 5900, "epoch": 12, "train_loss": 0.0023375635500997305, "cos_r_negg": [0.2285129725933075, 0.2835027277469635, 0.3267625570297241], "cos_innovation_negg": [0.2285129725933075, 0.2835027277469635, 0.3267625570297241], "cos_apical_negg": [0.0002898495877161622, 0.053974054753780365, 0.09298832714557648], "cos_Ac_negg": [0.5957195162773132, 0.6496959328651428, 0.6580698490142822], "r_norm": [0.16620704531669617, 0.1678336262702942, 0.16897913813591003]}, {"step": 6000, "epoch": 12, "train_loss": 0.0003156436432618648, "cos_r_negg": [0.22705024480819702, 0.28205057978630066, 0.3221222162246704], "cos_innovation_negg": [0.22705024480819702, 0.28205057978630066, 0.3221222162246704], "cos_apical_negg": [0.00018784520216286182, 0.05370461940765381, 0.08995052427053452], "cos_Ac_negg": [0.595751166343689, 0.650306224822998, 0.6576043963432312], "r_norm": [0.16127127408981323, 0.16293984651565552, 0.16411817073822021], "ldr": {"dL_sdil": -0.008749410510063171, "dL_bp": -0.0156887024641037, "ratio": 0.5576885998113693}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9732, "test_loss": 0.1357969973564148}, {"step": 6100, "epoch": 13, "train_loss": 0.00030620978213846684, "cos_r_negg": [0.2246711254119873, 0.2794528901576996, 0.3173590302467346], "cos_innovation_negg": [0.2246711254119873, 0.2794528901576996, 0.3173590302467346], "cos_apical_negg": [-0.0024985834024846554, 0.05318703502416611, 0.08709334582090378], "cos_Ac_negg": [0.5922064185142517, 0.6485128402709961, 0.6579561829566956], "r_norm": [0.15849639475345612, 0.15987026691436768, 0.16183032095432281]}, {"step": 6200, "epoch": 13, "train_loss": 0.000819559209048748, "cos_r_negg": [0.21680289506912231, 0.2714208662509918, 0.3083668351173401], "cos_innovation_negg": [0.21680289506912231, 0.2714208662509918, 0.3083668351173401], "cos_apical_negg": [-0.0016395982820540667, 0.05421179533004761, 0.08559107780456543], "cos_Ac_negg": [0.5904892683029175, 0.646056056022644, 0.6563012599945068], "r_norm": [0.15917962789535522, 0.1605096161365509, 0.1623694747686386]}, {"step": 6300, "epoch": 13, "train_loss": 0.0005211620591580868, "cos_r_negg": [0.21051853895187378, 0.266712486743927, 0.3031655251979828], "cos_innovation_negg": [0.21051853895187378, 0.266712486743927, 0.3031655251979828], "cos_apical_negg": [-0.00391730759292841, 0.05355990305542946, 0.08441944420337677], "cos_Ac_negg": [0.5890489816665649, 0.6451348066329956, 0.6550300121307373], "r_norm": [0.162738636136055, 0.16433613002300262, 0.16605722904205322]}, {"step": 6400, "epoch": 13, "train_loss": 0.00019245364819653332, "cos_r_negg": [0.21403810381889343, 0.2666715383529663, 0.3041558265686035], "cos_innovation_negg": [0.21403810381889343, 0.2666715383529663, 0.3041558265686035], "cos_apical_negg": [-0.002283968962728977, 0.05306035280227661, 0.08431465923786163], "cos_Ac_negg": [0.5907934904098511, 0.6469413638114929, 0.6580089926719666], "r_norm": [0.1597675085067749, 0.161129429936409, 0.1630009412765503]}, {"step": 6500, "epoch": 13, "train_loss": 8.894606435205787e-05, "cos_r_negg": [0.21055686473846436, 0.2640681266784668, 0.30202844738960266], "cos_innovation_negg": [0.21055686473846436, 0.2640681266784668, 0.30202844738960266], "cos_apical_negg": [-0.003650207072496414, 0.053804975003004074, 0.08419294655323029], "cos_Ac_negg": [0.5904040932655334, 0.6467540860176086, 0.6584591865539551], "r_norm": [0.15792900323867798, 0.15934449434280396, 0.16118767857551575], "ldr": {"dL_sdil": -0.008762657642364502, "dL_bp": -0.015807971358299255, "ratio": 0.554318922001593}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9732, "test_loss": 0.13811577005386352}, {"step": 6600, "epoch": 14, "train_loss": 0.0001483679807279259, "cos_r_negg": [0.20789846777915955, 0.2618699073791504, 0.3022313117980957], "cos_innovation_negg": [0.20789846777915955, 0.2618699073791504, 0.3022313117980957], "cos_apical_negg": [-0.002257916610687971, 0.053595833480358124, 0.08734874427318573], "cos_Ac_negg": [0.589479923248291, 0.6461293697357178, 0.6584813594818115], "r_norm": [0.15781164169311523, 0.15928754210472107, 0.16065174341201782]}, {"step": 6700, "epoch": 14, "train_loss": 0.0001985140552278608, "cos_r_negg": [0.20521223545074463, 0.25920915603637695, 0.29712140560150146], "cos_innovation_negg": [0.20521223545074463, 0.25920915603637695, 0.29712140560150146], "cos_apical_negg": [-0.0038474048487842083, 0.052335552871227264, 0.08340521156787872], "cos_Ac_negg": [0.58717942237854, 0.6465771198272705, 0.657573938369751], "r_norm": [0.15585049986839294, 0.15715712308883667, 0.15876515209674835]}, {"step": 6800, "epoch": 14, "train_loss": 0.002169717103242874, "cos_r_negg": [0.2027571201324463, 0.2553759217262268, 0.2935708165168762], "cos_innovation_negg": [0.2027571201324463, 0.2553759217262268, 0.2935708165168762], "cos_apical_negg": [-0.0022205498535186052, 0.05251191556453705, 0.08286882191896439], "cos_Ac_negg": [0.5884904265403748, 0.645618200302124, 0.6573837995529175], "r_norm": [0.15644827485084534, 0.1578161120414734, 0.15946391224861145]}, {"step": 6900, "epoch": 14, "train_loss": 0.00026743928901851177, "cos_r_negg": [0.19792090356349945, 0.25146740674972534, 0.2899780571460724], "cos_innovation_negg": [0.19792090356349945, 0.25146740674972534, 0.2899780571460724], "cos_apical_negg": [-0.0020037228241562843, 0.052818696945905685, 0.08272625505924225], "cos_Ac_negg": [0.5900148153305054, 0.6462239623069763, 0.6577309370040894], "r_norm": [0.15712875127792358, 0.15858200192451477, 0.1602741777896881]}, {"step": 7000, "epoch": 14, "train_loss": 0.000929106492549181, "cos_r_negg": [0.19869264960289001, 0.25120824575424194, 0.28751811385154724], "cos_innovation_negg": [0.19869264960289001, 0.25120824575424194, 0.28751811385154724], "cos_apical_negg": [-0.0020008564461022615, 0.05232146382331848, 0.08212249726057053], "cos_Ac_negg": [0.5898680686950684, 0.6445186138153076, 0.6583946943283081], "r_norm": [0.15886053442955017, 0.16022218763828278, 0.16218824684619904], "ldr": {"dL_sdil": -0.008912354707717896, "dL_bp": -0.01609010249376297, "ratio": 0.553902917099043}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9728, "test_loss": 0.14043981161117552}], "final": {"test_acc": 0.9728, "test_loss": 0.14043981161117552, "wall_s": 24.184848070144653, "cos_r_negg": [0.19753050804138184, 0.25003182888031006, 0.28698357939720154], "cos_innovation_negg": [0.19753050804138184, 0.25003182888031006, 0.28698357939720154], "cos_apical_negg": [-0.0024302112869918346, 0.051940012723207474, 0.08151642978191376], "cos_Ac_negg": [0.5888445377349854, 0.6440334916114807, 0.6589258909225464]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_residual_s3.json b/results/nuis_ctrl_v1_rho0p2_residual_s3.json new file mode 100644 index 0000000..d0f7850 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_residual_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_residual_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.01401346456259489, 0.01371360756456852, 0.006088063586503267], "cos_innovation_negg": [-0.01401346456259489, 0.01371360756456852, 0.006088063586503267], "cos_apical_negg": [-0.01633278653025627, 0.010381842963397503, 0.002567625604569912], "cos_Ac_negg": [-0.014014208689332008, 0.013713478110730648, 0.006087634712457657], "r_norm": [4.854386329650879, 4.749161720275879, 4.848539352416992], "ldr": {"dL_sdil": -0.00576019287109375, "dL_bp": -0.23061609268188477, "ratio": 0.02497741074400842}}, {"step": 100, "epoch": 0, "train_loss": 0.18873512744903564, "cos_r_negg": [0.42551085352897644, 0.3603532612323761, 0.31459999084472656], "cos_innovation_negg": [0.42551085352897644, 0.3603532612323761, 0.31459999084472656], "cos_apical_negg": [0.11449956893920898, 0.11613072454929352, 0.14878784120082855], "cos_Ac_negg": [0.42719539999961853, 0.36063358187675476, 0.3139951825141907], "r_norm": [0.8626118898391724, 0.8500277996063232, 0.8591506481170654]}, {"step": 200, "epoch": 0, "train_loss": 0.2459070384502411, "cos_r_negg": [0.48045146465301514, 0.43604618310928345, 0.35125911235809326], "cos_innovation_negg": [0.48045146465301514, 0.43604618310928345, 0.35125911235809326], "cos_apical_negg": [0.10844692587852478, 0.11874005943536758, 0.1324051320552826], "cos_Ac_negg": [0.4816785156726837, 0.4371064305305481, 0.3505333960056305], "r_norm": [0.7453992366790771, 0.7316138744354248, 0.7445143461227417]}, {"step": 300, "epoch": 0, "train_loss": 0.1282867193222046, "cos_r_negg": [0.49072495102882385, 0.45970141887664795, 0.38517892360687256], "cos_innovation_negg": [0.49072495102882385, 0.45970141887664795, 0.38517892360687256], "cos_apical_negg": [0.09590651839971542, 0.11035208404064178, 0.13731591403484344], "cos_Ac_negg": [0.4915056526660919, 0.4599386751651764, 0.3853808343410492], "r_norm": [0.5990716218948364, 0.5922057628631592, 0.598983645439148]}, {"step": 400, "epoch": 0, "train_loss": 0.1547357439994812, "cos_r_negg": [0.5252184867858887, 0.49495863914489746, 0.41902077198028564], "cos_innovation_negg": [0.5252184867858887, 0.49495863914489746, 0.41902077198028564], "cos_apical_negg": [0.08515290915966034, 0.1074976921081543, 0.13469132781028748], "cos_Ac_negg": [0.5259838104248047, 0.49456217885017395, 0.41876617074012756], "r_norm": [0.5662006735801697, 0.5600329637527466, 0.5683958530426025]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9467, "test_loss": 0.18566118240356444}, {"step": 500, "epoch": 1, "train_loss": 0.2022823989391327, "cos_r_negg": [0.5014853477478027, 0.49486565589904785, 0.4415689706802368], "cos_innovation_negg": [0.5014853477478027, 0.49486565589904785, 0.4415689706802368], "cos_apical_negg": [0.07622790336608887, 0.09425497055053711, 0.14676497876644135], "cos_Ac_negg": [0.5032318830490112, 0.4970094859600067, 0.4418157935142517], "r_norm": [0.5341676473617554, 0.5255112648010254, 0.5302309393882751], "ldr": {"dL_sdil": -0.015126332640647888, "dL_bp": -0.013028159737586975, "ratio": 1.1610490618262506}}, {"step": 600, "epoch": 1, "train_loss": 0.09608648717403412, "cos_r_negg": [0.5118353962898254, 0.5069502592086792, 0.4366770386695862], "cos_innovation_negg": [0.5118353962898254, 0.5069502592086792, 0.4366770386695862], "cos_apical_negg": [0.06319546699523926, 0.08845227211713791, 0.12924151122570038], "cos_Ac_negg": [0.5148888826370239, 0.5091942548751831, 0.4363383948802948], "r_norm": [0.4012887477874756, 0.39506545662879944, 0.4009026885032654]}, {"step": 700, "epoch": 1, "train_loss": 0.1620636284351349, "cos_r_negg": [0.5468978881835938, 0.5389772057533264, 0.4663362205028534], "cos_innovation_negg": [0.5468978881835938, 0.5389772057533264, 0.4663362205028534], "cos_apical_negg": [0.06697755306959152, 0.09641847759485245, 0.13771817088127136], "cos_Ac_negg": [0.5502389669418335, 0.5403789281845093, 0.4665992856025696], "r_norm": [0.4150056838989258, 0.40969038009643555, 0.4158512055873871]}, {"step": 800, "epoch": 1, "train_loss": 0.14280110597610474, "cos_r_negg": [0.5250078439712524, 0.5242472887039185, 0.4713127613067627], "cos_innovation_negg": [0.5250078439712524, 0.5242472887039185, 0.4713127613067627], "cos_apical_negg": [0.050785697996616364, 0.08504807949066162, 0.1308087259531021], "cos_Ac_negg": [0.5275696516036987, 0.5260343551635742, 0.4711673855781555], "r_norm": [0.34762388467788696, 0.34460484981536865, 0.3470115661621094]}, {"step": 900, "epoch": 1, "train_loss": 0.22257229685783386, "cos_r_negg": [0.542015552520752, 0.5338290333747864, 0.47017067670822144], "cos_innovation_negg": [0.542015552520752, 0.5338290333747864, 0.47017067670822144], "cos_apical_negg": [0.08236466348171234, 0.08128423243761063, 0.1286977231502533], "cos_Ac_negg": [0.5421379804611206, 0.5341194868087769, 0.469769686460495], "r_norm": [0.42938604950904846, 0.4241929054260254, 0.42841291427612305]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9578, "test_loss": 0.15420068130493164}, {"step": 1000, "epoch": 2, "train_loss": 0.036834731698036194, "cos_r_negg": [0.5252987146377563, 0.5434536337852478, 0.4803529679775238], "cos_innovation_negg": [0.5252987146377563, 0.5434536337852478, 0.4803529679775238], "cos_apical_negg": [0.055854231119155884, 0.08249439299106598, 0.11783044785261154], "cos_Ac_negg": [0.5273569822311401, 0.544925332069397, 0.4794422686100006], "r_norm": [0.33082473278045654, 0.3307965397834778, 0.3310631215572357], "ldr": {"dL_sdil": -0.006736010313034058, "dL_bp": -0.006280556321144104, "ratio": 1.0725180969011652}}, {"step": 1100, "epoch": 2, "train_loss": 0.1352086365222931, "cos_r_negg": [0.5510923862457275, 0.5639913082122803, 0.5086607933044434], "cos_innovation_negg": [0.5510923862457275, 0.5639913082122803, 0.5086607933044434], "cos_apical_negg": [0.06886047124862671, 0.08101245760917664, 0.12585797905921936], "cos_Ac_negg": [0.5541568994522095, 0.5669642090797424, 0.50771164894104], "r_norm": [0.3794521689414978, 0.37344086170196533, 0.37624090909957886]}, {"step": 1200, "epoch": 2, "train_loss": 0.1536949872970581, "cos_r_negg": [0.537842333316803, 0.560580313205719, 0.5249142646789551], "cos_innovation_negg": [0.537842333316803, 0.560580313205719, 0.5249142646789551], "cos_apical_negg": [0.05177416652441025, 0.06955734640359879, 0.13303402066230774], "cos_Ac_negg": [0.5428236126899719, 0.5652302503585815, 0.5233486890792847], "r_norm": [0.3386765718460083, 0.33176708221435547, 0.33597254753112793]}, {"step": 1300, "epoch": 2, "train_loss": 0.10706286132335663, "cos_r_negg": [0.5713360905647278, 0.5788295269012451, 0.5307925343513489], "cos_innovation_negg": [0.5713360905647278, 0.5788295269012451, 0.5307925343513489], "cos_apical_negg": [0.05037011206150055, 0.06893859058618546, 0.12395511567592621], "cos_Ac_negg": [0.5746093988418579, 0.581193745136261, 0.5294284820556641], "r_norm": [0.2783706784248352, 0.2770356237888336, 0.2780460715293884]}, {"step": 1400, "epoch": 2, "train_loss": 0.1364029049873352, "cos_r_negg": [0.5651035904884338, 0.5812767148017883, 0.523108720779419], "cos_innovation_negg": [0.5651035904884338, 0.5812767148017883, 0.523108720779419], "cos_apical_negg": [0.04892797768115997, 0.0780453309416771, 0.1301746368408203], "cos_Ac_negg": [0.5700987577438354, 0.5863732099533081, 0.5222785472869873], "r_norm": [0.28419798612594604, 0.28103306889533997, 0.2831452786922455]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9642, "test_loss": 0.11836729297637939}, {"step": 1500, "epoch": 3, "train_loss": 0.038324855268001556, "cos_r_negg": [0.5681567192077637, 0.5817520618438721, 0.5452873706817627], "cos_innovation_negg": [0.5681567192077637, 0.5817520618438721, 0.5452873706817627], "cos_apical_negg": [0.04089711979031563, 0.061245303601026535, 0.11822941154241562], "cos_Ac_negg": [0.5736209750175476, 0.5875518321990967, 0.545030951499939], "r_norm": [0.24270948767662048, 0.23855535686016083, 0.23999647796154022], "ldr": {"dL_sdil": -0.0048094242811203, "dL_bp": -0.004704371094703674, "ratio": 1.0223309735354207}}, {"step": 1600, "epoch": 3, "train_loss": 0.11071884632110596, "cos_r_negg": [0.573799192905426, 0.5846590995788574, 0.5462180376052856], "cos_innovation_negg": [0.573799192905426, 0.5846590995788574, 0.5462180376052856], "cos_apical_negg": [0.049994248896837234, 0.06502309441566467, 0.1271304488182068], "cos_Ac_negg": [0.5799444317817688, 0.5919026136398315, 0.5455365180969238], "r_norm": [0.27009037137031555, 0.26588764786720276, 0.26728346943855286]}, {"step": 1700, "epoch": 3, "train_loss": 0.08539792895317078, "cos_r_negg": [0.560289740562439, 0.5728726387023926, 0.5539113283157349], "cos_innovation_negg": [0.560289740562439, 0.5728726387023926, 0.5539113283157349], "cos_apical_negg": [0.0549091175198555, 0.06651626527309418, 0.1194741278886795], "cos_Ac_negg": [0.571294367313385, 0.5842162370681763, 0.5567351579666138], "r_norm": [0.26270005106925964, 0.2590261399745941, 0.2589811086654663]}, {"step": 1800, "epoch": 3, "train_loss": 0.04495394974946976, "cos_r_negg": [0.5494127869606018, 0.5575644373893738, 0.5537464618682861], "cos_innovation_negg": [0.5494127869606018, 0.5575644373893738, 0.5537464618682861], "cos_apical_negg": [0.03702382743358612, 0.05269481986761093, 0.12010857462882996], "cos_Ac_negg": [0.5647628307342529, 0.5746102929115295, 0.5641068816184998], "r_norm": [0.25814348459243774, 0.25500357151031494, 0.2557433247566223]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9646, "test_loss": 0.11824163208007812}, {"step": 1900, "epoch": 4, "train_loss": 0.04704385623335838, "cos_r_negg": [0.5639058947563171, 0.5820882320404053, 0.5742977857589722], "cos_innovation_negg": [0.5639058947563171, 0.5820882320404053, 0.5742977857589722], "cos_apical_negg": [0.04958593472838402, 0.07463567703962326, 0.12237516790628433], "cos_Ac_negg": [0.573023796081543, 0.5931601524353027, 0.5787421464920044], "r_norm": [0.3080936670303345, 0.30260777473449707, 0.3050350248813629]}, {"step": 2000, "epoch": 4, "train_loss": 0.1090206578373909, "cos_r_negg": [0.5349593758583069, 0.546436071395874, 0.5513330698013306], "cos_innovation_negg": [0.5349593758583069, 0.546436071395874, 0.5513330698013306], "cos_apical_negg": [0.035840194672346115, 0.05269690230488777, 0.11402727663516998], "cos_Ac_negg": [0.5707863569259644, 0.5776305198669434, 0.568793535232544], "r_norm": [0.22153672575950623, 0.21603810787200928, 0.21736770868301392], "ldr": {"dL_sdil": -0.006291545927524567, "dL_bp": -0.00635499507188797, "ratio": 0.9900158625387331}}, {"step": 2100, "epoch": 4, "train_loss": 0.023871950805187225, "cos_r_negg": [0.5592578649520874, 0.5849747061729431, 0.5727667808532715], "cos_innovation_negg": [0.5592578649520874, 0.5849747061729431, 0.5727667808532715], "cos_apical_negg": [0.026188362389802933, 0.060945652425289154, 0.12014678120613098], "cos_Ac_negg": [0.5772706270217896, 0.6038010120391846, 0.5796332359313965], "r_norm": [0.23918426036834717, 0.23436924815177917, 0.23573587834835052]}, {"step": 2200, "epoch": 4, "train_loss": 0.045306432992219925, "cos_r_negg": [0.5652357339859009, 0.5907092094421387, 0.5843456387519836], "cos_innovation_negg": [0.5652357339859009, 0.5907092094421387, 0.5843456387519836], "cos_apical_negg": [0.023994604125618935, 0.05470263212919235, 0.11344252526760101], "cos_Ac_negg": [0.5840235948562622, 0.6090487837791443, 0.5904650092124939], "r_norm": [0.22215567529201508, 0.21861065924167633, 0.21919593214988708]}, {"step": 2300, "epoch": 4, "train_loss": 0.18277393281459808, "cos_r_negg": [0.5552376508712769, 0.5786513090133667, 0.5839237570762634], "cos_innovation_negg": [0.5552376508712769, 0.5786513090133667, 0.5839237570762634], "cos_apical_negg": [0.020443785935640335, 0.041437696665525436, 0.11584413051605225], "cos_Ac_negg": [0.5809895992279053, 0.60596764087677, 0.5975843667984009], "r_norm": [0.19766755402088165, 0.1951572448015213, 0.19576910138130188]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9697, "test_loss": 0.11493425922393799}, {"step": 2400, "epoch": 5, "train_loss": 0.06708382815122604, "cos_r_negg": [0.5531058311462402, 0.5770075917243958, 0.5808913707733154], "cos_innovation_negg": [0.5531058311462402, 0.5770075917243958, 0.5808913707733154], "cos_apical_negg": [0.023905128240585327, 0.04676685109734535, 0.10950885713100433], "cos_Ac_negg": [0.5943174958229065, 0.6174683570861816, 0.6111822128295898], "r_norm": [0.181976318359375, 0.17854291200637817, 0.1797497570514679]}, {"step": 2500, "epoch": 5, "train_loss": 0.03424198925495148, "cos_r_negg": [0.5054612159729004, 0.5498422384262085, 0.5621560215950012], "cos_innovation_negg": [0.5054612159729004, 0.5498422384262085, 0.5621560215950012], "cos_apical_negg": [0.03350185975432396, 0.049942366778850555, 0.11881458014249802], "cos_Ac_negg": [0.5766274333000183, 0.6187105178833008, 0.605825662612915], "r_norm": [0.20842283964157104, 0.20446133613586426, 0.20585647225379944], "ldr": {"dL_sdil": -0.005982600152492523, "dL_bp": -0.006830863654613495, "ratio": 0.875818996687474}}, {"step": 2600, "epoch": 5, "train_loss": 0.057028595358133316, "cos_r_negg": [0.5284832715988159, 0.5672512054443359, 0.5801628828048706], "cos_innovation_negg": [0.5284832715988159, 0.5672512054443359, 0.5801628828048706], "cos_apical_negg": [0.032390762120485306, 0.04888254776597023, 0.11695471405982971], "cos_Ac_negg": [0.59566330909729, 0.6274288892745972, 0.6236974000930786], "r_norm": [0.20026922225952148, 0.19785061478614807, 0.19861774146556854]}, {"step": 2700, "epoch": 5, "train_loss": 0.04744546115398407, "cos_r_negg": [0.5307871699333191, 0.5803029537200928, 0.5932604074478149], "cos_innovation_negg": [0.5307871699333191, 0.5803029537200928, 0.5932604074478149], "cos_apical_negg": [0.018124477937817574, 0.053931158035993576, 0.12388523668050766], "cos_Ac_negg": [0.5922767519950867, 0.6326730251312256, 0.6278306245803833], "r_norm": [0.2167394459247589, 0.21486571431159973, 0.21571873128414154]}, {"step": 2800, "epoch": 5, "train_loss": 0.07268582284450531, "cos_r_negg": [0.5360569357872009, 0.5850205421447754, 0.5884784460067749], "cos_innovation_negg": [0.5360569357872009, 0.5850205421447754, 0.5884784460067749], "cos_apical_negg": [0.02157643996179104, 0.05472957715392113, 0.12207011133432388], "cos_Ac_negg": [0.5895681977272034, 0.6297746896743774, 0.6170710325241089], "r_norm": [0.2280716747045517, 0.2248237282037735, 0.22618234157562256]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9708, "test_loss": 0.11395362854003906}, {"step": 2900, "epoch": 6, "train_loss": 0.006686285138130188, "cos_r_negg": [0.51558518409729, 0.5628201961517334, 0.572902262210846], "cos_innovation_negg": [0.51558518409729, 0.5628201961517334, 0.572902262210846], "cos_apical_negg": [0.025216661393642426, 0.04977145791053772, 0.12181040644645691], "cos_Ac_negg": [0.5843631625175476, 0.6313927173614502, 0.6231183409690857], "r_norm": [0.23633728921413422, 0.23144465684890747, 0.23330065608024597]}, {"step": 3000, "epoch": 6, "train_loss": 0.0894983783364296, "cos_r_negg": [0.505083441734314, 0.5472036004066467, 0.5636588335037231], "cos_innovation_negg": [0.505083441734314, 0.5472036004066467, 0.5636588335037231], "cos_apical_negg": [0.016221128404140472, 0.03905455768108368, 0.11353820562362671], "cos_Ac_negg": [0.590485692024231, 0.6331952810287476, 0.6208517551422119], "r_norm": [0.19714771211147308, 0.19382169842720032, 0.19380110502243042], "ldr": {"dL_sdil": -0.005583271384239197, "dL_bp": -0.006370678544044495, "ratio": 0.8764013669248168}}, {"step": 3100, "epoch": 6, "train_loss": 0.024059748277068138, "cos_r_negg": [0.48229914903640747, 0.5265419483184814, 0.5552312135696411], "cos_innovation_negg": [0.48229914903640747, 0.5265419483184814, 0.5552312135696411], "cos_apical_negg": [0.01646444946527481, 0.04031222686171532, 0.11380480229854584], "cos_Ac_negg": [0.5978851914405823, 0.6291283965110779, 0.6335819959640503], "r_norm": [0.1687355637550354, 0.164513498544693, 0.16626185178756714]}, {"step": 3200, "epoch": 6, "train_loss": 0.06095162034034729, "cos_r_negg": [0.44403892755508423, 0.5013418197631836, 0.5414658784866333], "cos_innovation_negg": [0.44403892755508423, 0.5013418197631836, 0.5414658784866333], "cos_apical_negg": [0.005028644576668739, 0.05140520632266998, 0.1214417964220047], "cos_Ac_negg": [0.5862263441085815, 0.6361229419708252, 0.639459490776062], "r_norm": [0.1771746277809143, 0.17487776279449463, 0.1754063069820404]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9691, "test_loss": 0.12765268573760985}, {"step": 3300, "epoch": 7, "train_loss": 0.023883402347564697, "cos_r_negg": [0.4372710585594177, 0.49594300985336304, 0.5368139743804932], "cos_innovation_negg": [0.4372710585594177, 0.49594300985336304, 0.5368139743804932], "cos_apical_negg": [0.004016466438770294, 0.04841306433081627, 0.10837838053703308], "cos_Ac_negg": [0.5841434001922607, 0.6352051496505737, 0.6438031196594238], "r_norm": [0.17338460683822632, 0.17009533941745758, 0.17204077541828156]}, {"step": 3400, "epoch": 7, "train_loss": 0.011563774198293686, "cos_r_negg": [0.4032028019428253, 0.4476282000541687, 0.49372589588165283], "cos_innovation_negg": [0.4032028019428253, 0.4476282000541687, 0.49372589588165283], "cos_apical_negg": [0.02277015522122383, 0.0464412160217762, 0.11609035730361938], "cos_Ac_negg": [0.5841609239578247, 0.6342027187347412, 0.6419724225997925], "r_norm": [0.2020149677991867, 0.19642813503742218, 0.19873930513858795]}, {"step": 3500, "epoch": 7, "train_loss": 0.013702900148928165, "cos_r_negg": [0.41426321864128113, 0.454189658164978, 0.5024068355560303], "cos_innovation_negg": [0.41426321864128113, 0.454189658164978, 0.5024068355560303], "cos_apical_negg": [0.009891992434859276, 0.03601841628551483, 0.11002354323863983], "cos_Ac_negg": [0.6021798849105835, 0.6395878791809082, 0.6560165882110596], "r_norm": [0.17867809534072876, 0.17483720183372498, 0.17670764029026031], "ldr": {"dL_sdil": -0.006468772888183594, "dL_bp": -0.00890311598777771, "ratio": 0.7265740328514187}}, {"step": 3600, "epoch": 7, "train_loss": 0.0031365768518298864, "cos_r_negg": [0.40761950612068176, 0.4544501006603241, 0.49897146224975586], "cos_innovation_negg": [0.40761950612068176, 0.4544501006603241, 0.49897146224975586], "cos_apical_negg": [0.015406962484121323, 0.03500514477491379, 0.10255996882915497], "cos_Ac_negg": [0.5953332185745239, 0.6431121230125427, 0.6512629985809326], "r_norm": [0.16711248457431793, 0.16384246945381165, 0.165050208568573]}, {"step": 3700, "epoch": 7, "train_loss": 0.011196072213351727, "cos_r_negg": [0.43130093812942505, 0.4804968237876892, 0.5159401297569275], "cos_innovation_negg": [0.43130093812942505, 0.4804968237876892, 0.5159401297569275], "cos_apical_negg": [0.011967116966843605, 0.042917899787425995, 0.1083512157201767], "cos_Ac_negg": [0.6050165295600891, 0.6480522751808167, 0.6599148511886597], "r_norm": [0.16527262330055237, 0.16260609030723572, 0.1636393964290619]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9733, "test_loss": 0.11964113779067993}, {"step": 3800, "epoch": 8, "train_loss": 0.0023065307177603245, "cos_r_negg": [0.4035950005054474, 0.4397372603416443, 0.49234071373939514], "cos_innovation_negg": [0.4035950005054474, 0.4397372603416443, 0.49234071373939514], "cos_apical_negg": [0.013022277504205704, 0.03663323447108269, 0.11164499819278717], "cos_Ac_negg": [0.5987977981567383, 0.6396538019180298, 0.660892128944397], "r_norm": [0.16224171221256256, 0.16038155555725098, 0.16063547134399414]}, {"step": 3900, "epoch": 8, "train_loss": 0.017203664407134056, "cos_r_negg": [0.3728996217250824, 0.41909801959991455, 0.4681956171989441], "cos_innovation_negg": [0.3728996217250824, 0.41909801959991455, 0.4681956171989441], "cos_apical_negg": [0.013784561306238174, 0.037006475031375885, 0.10990840941667557], "cos_Ac_negg": [0.5922143459320068, 0.642852783203125, 0.6614621877670288], "r_norm": [0.15772667527198792, 0.1537609100341797, 0.15576636791229248]}, {"step": 4000, "epoch": 8, "train_loss": 0.00283118337392807, "cos_r_negg": [0.3320258855819702, 0.39198315143585205, 0.44947540760040283], "cos_innovation_negg": [0.3320258855819702, 0.39198315143585205, 0.44947540760040283], "cos_apical_negg": [0.002690427703782916, 0.040405161678791046, 0.1105620414018631], "cos_Ac_negg": [0.5902037620544434, 0.6408340930938721, 0.66427081823349], "r_norm": [0.14475294947624207, 0.1420484483242035, 0.1434231400489807], "ldr": {"dL_sdil": -0.0046197399497032166, "dL_bp": -0.006421804428100586, "ratio": 0.7193834694635233}}, {"step": 4100, "epoch": 8, "train_loss": 0.0056511638686060905, "cos_r_negg": [0.36505937576293945, 0.42459988594055176, 0.4812759757041931], "cos_innovation_negg": [0.36505937576293945, 0.42459988594055176, 0.4812759757041931], "cos_apical_negg": [0.004104168154299259, 0.04041241854429245, 0.11180675029754639], "cos_Ac_negg": [0.5889151096343994, 0.6453028321266174, 0.6706963181495667], "r_norm": [0.1615711897611618, 0.15812571346759796, 0.15983152389526367]}, {"step": 4200, "epoch": 8, "train_loss": 0.01308977510780096, "cos_r_negg": [0.3646843433380127, 0.4085808992385864, 0.45853450894355774], "cos_innovation_negg": [0.3646843433380127, 0.4085808992385864, 0.45853450894355774], "cos_apical_negg": [0.007420526351779699, 0.04285050928592682, 0.10662416368722916], "cos_Ac_negg": [0.6015160083770752, 0.650770902633667, 0.6693019866943359], "r_norm": [0.14816445112228394, 0.14529353380203247, 0.14685791730880737]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9734, "test_loss": 0.12385242824554443}, {"step": 4300, "epoch": 9, "train_loss": 0.009149294346570969, "cos_r_negg": [0.3273484408855438, 0.3603981137275696, 0.41702908277511597], "cos_innovation_negg": [0.3273484408855438, 0.3603981137275696, 0.41702908277511597], "cos_apical_negg": [0.011426137760281563, 0.03711375594139099, 0.10309694707393646], "cos_Ac_negg": [0.5904088020324707, 0.6431702375411987, 0.6659565567970276], "r_norm": [0.164593905210495, 0.16200922429561615, 0.16326048970222473]}, {"step": 4400, "epoch": 9, "train_loss": 0.0012302198447287083, "cos_r_negg": [0.34810036420822144, 0.3918251693248749, 0.44217023253440857], "cos_innovation_negg": [0.34810036420822144, 0.3918251693248749, 0.44217023253440857], "cos_apical_negg": [0.005247277207672596, 0.037915945053100586, 0.10195288062095642], "cos_Ac_negg": [0.5923473238945007, 0.6484144330024719, 0.6756606101989746], "r_norm": [0.13819624483585358, 0.13477015495300293, 0.13712504506111145]}, {"step": 4500, "epoch": 9, "train_loss": 0.03266465291380882, "cos_r_negg": [0.3250414729118347, 0.35153859853744507, 0.405872642993927], "cos_innovation_negg": [0.3250414729118347, 0.35153859853744507, 0.405872642993927], "cos_apical_negg": [0.00648121815174818, 0.03344428166747093, 0.09688430279493332], "cos_Ac_negg": [0.593754768371582, 0.637917160987854, 0.6645973920822144], "r_norm": [0.1476287692785263, 0.14393720030784607, 0.14630907773971558], "ldr": {"dL_sdil": -0.0060315802693367004, "dL_bp": -0.008732922375202179, "ratio": 0.6906714625637631}}, {"step": 4600, "epoch": 9, "train_loss": 0.0008049164898693562, "cos_r_negg": [0.33393365144729614, 0.37126344442367554, 0.4240191578865051], "cos_innovation_negg": [0.33393365144729614, 0.37126344442367554, 0.4240191578865051], "cos_apical_negg": [0.013297813013195992, 0.035311177372932434, 0.09767794609069824], "cos_Ac_negg": [0.5870927572250366, 0.6417601108551025, 0.6765743494033813], "r_norm": [0.15512298047542572, 0.15194562077522278, 0.1534360647201538]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9729, "test_loss": 0.12904755640029908}, {"step": 4700, "epoch": 10, "train_loss": 0.0050363666377961636, "cos_r_negg": [0.2971585690975189, 0.3426578938961029, 0.3923370838165283], "cos_innovation_negg": [0.2971585690975189, 0.3426578938961029, 0.3923370838165283], "cos_apical_negg": [0.006049447692930698, 0.0372016504406929, 0.09562559425830841], "cos_Ac_negg": [0.582510232925415, 0.6433707475662231, 0.6715540885925293], "r_norm": [0.14715415239334106, 0.14434099197387695, 0.14648661017417908]}, {"step": 4800, "epoch": 10, "train_loss": 0.005406802985817194, "cos_r_negg": [0.28977954387664795, 0.3338377773761749, 0.38440027832984924], "cos_innovation_negg": [0.28977954387664795, 0.3338377773761749, 0.38440027832984924], "cos_apical_negg": [0.00969766266644001, 0.037736676633358, 0.09605565667152405], "cos_Ac_negg": [0.5797987580299377, 0.6394490003585815, 0.6709378957748413], "r_norm": [0.14719560742378235, 0.14388829469680786, 0.14618919789791107]}, {"step": 4900, "epoch": 10, "train_loss": 0.0016784570179879665, "cos_r_negg": [0.29306820034980774, 0.33002734184265137, 0.3801448345184326], "cos_innovation_negg": [0.29306820034980774, 0.33002734184265137, 0.3801448345184326], "cos_apical_negg": [0.008315379731357098, 0.03247849643230438, 0.0929107517004013], "cos_Ac_negg": [0.5799634456634521, 0.637857973575592, 0.668197512626648], "r_norm": [0.13963955640792847, 0.13658124208450317, 0.1389416605234146]}, {"step": 5000, "epoch": 10, "train_loss": 0.0007381493342109025, "cos_r_negg": [0.27754515409469604, 0.31434905529022217, 0.3627849221229553], "cos_innovation_negg": [0.27754515409469604, 0.31434905529022217, 0.3627849221229553], "cos_apical_negg": [0.00608882075175643, 0.03207935392856598, 0.09047340601682663], "cos_Ac_negg": [0.576554536819458, 0.639495313167572, 0.6658531427383423], "r_norm": [0.14053618907928467, 0.13784439861774445, 0.14010469615459442], "ldr": {"dL_sdil": -0.006616033613681793, "dL_bp": -0.010654069483280182, "ratio": 0.6209865276422849}}, {"step": 5100, "epoch": 10, "train_loss": 0.0026385232340544462, "cos_r_negg": [0.28332141041755676, 0.318595290184021, 0.3708907663822174], "cos_innovation_negg": [0.28332141041755676, 0.318595290184021, 0.3708907663822174], "cos_apical_negg": [0.00385945662856102, 0.03137340396642685, 0.0917658880352974], "cos_Ac_negg": [0.5841526985168457, 0.6423323154449463, 0.6733631491661072], "r_norm": [0.14305317401885986, 0.14065806567668915, 0.14249151945114136]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9737, "test_loss": 0.1326816659927368}, {"step": 5200, "epoch": 11, "train_loss": 0.0011589385103434324, "cos_r_negg": [0.278288334608078, 0.3090996742248535, 0.36126548051834106], "cos_innovation_negg": [0.278288334608078, 0.3090996742248535, 0.36126548051834106], "cos_apical_negg": [0.008557699620723724, 0.03427043557167053, 0.08963680267333984], "cos_Ac_negg": [0.5796408653259277, 0.638587474822998, 0.6719264984130859], "r_norm": [0.134597510099411, 0.13201984763145447, 0.13406230509281158]}, {"step": 5300, "epoch": 11, "train_loss": 0.00224539777263999, "cos_r_negg": [0.26544421911239624, 0.30028676986694336, 0.34926825761795044], "cos_innovation_negg": [0.26544421911239624, 0.30028676986694336, 0.34926825761795044], "cos_apical_negg": [0.0069392225705087185, 0.030967628583312035, 0.0874195545911789], "cos_Ac_negg": [0.575717031955719, 0.6370882987976074, 0.6674982309341431], "r_norm": [0.13563451170921326, 0.13257160782814026, 0.1347551941871643]}, {"step": 5400, "epoch": 11, "train_loss": 0.002859623171389103, "cos_r_negg": [0.2713771462440491, 0.3025330901145935, 0.3500608205795288], "cos_innovation_negg": [0.2713771462440491, 0.3025330901145935, 0.3500608205795288], "cos_apical_negg": [0.007332724519073963, 0.029432091861963272, 0.08250143378973007], "cos_Ac_negg": [0.5809556245803833, 0.6392914652824402, 0.6695345640182495], "r_norm": [0.13454987108707428, 0.1319369077682495, 0.13394761085510254]}, {"step": 5500, "epoch": 11, "train_loss": 0.0011915442300960422, "cos_r_negg": [0.2577810287475586, 0.29153379797935486, 0.34306567907333374], "cos_innovation_negg": [0.2577810287475586, 0.29153379797935486, 0.34306567907333374], "cos_apical_negg": [0.004283071495592594, 0.03088618442416191, 0.0867825299501419], "cos_Ac_negg": [0.5771710872650146, 0.6397589445114136, 0.6721286773681641], "r_norm": [0.13539481163024902, 0.13255712389945984, 0.13476434350013733], "ldr": {"dL_sdil": -0.005841605365276337, "dL_bp": -0.009094223380088806, "ratio": 0.6423424102454027}}, {"step": 5600, "epoch": 11, "train_loss": 0.0009290726156905293, "cos_r_negg": [0.2523663640022278, 0.28946352005004883, 0.3401210308074951], "cos_innovation_negg": [0.2523663640022278, 0.28946352005004883, 0.3401210308074951], "cos_apical_negg": [0.0022307210601866245, 0.03210269659757614, 0.08586939424276352], "cos_Ac_negg": [0.5747398138046265, 0.6356382369995117, 0.671718955039978], "r_norm": [0.13824433088302612, 0.13527542352676392, 0.13748770952224731]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9738, "test_loss": 0.13491204013824462}, {"step": 5700, "epoch": 12, "train_loss": 0.001086224801838398, "cos_r_negg": [0.24984243512153625, 0.28309738636016846, 0.33308276534080505], "cos_innovation_negg": [0.24984243512153625, 0.28309738636016846, 0.33308276534080505], "cos_apical_negg": [0.007476150523871183, 0.03297806903719902, 0.08509723842144012], "cos_Ac_negg": [0.5744707584381104, 0.635699987411499, 0.6720552444458008], "r_norm": [0.1377202570438385, 0.13466627895832062, 0.13708040118217468]}, {"step": 5800, "epoch": 12, "train_loss": 0.00036634469870477915, "cos_r_negg": [0.24313117563724518, 0.27526184916496277, 0.3211178183555603], "cos_innovation_negg": [0.24313117563724518, 0.27526184916496277, 0.3211178183555603], "cos_apical_negg": [0.005894692614674568, 0.030842486768960953, 0.07910270243883133], "cos_Ac_negg": [0.5708178877830505, 0.6340307593345642, 0.6696828603744507], "r_norm": [0.13602876663208008, 0.1329473853111267, 0.13534313440322876]}, {"step": 5900, "epoch": 12, "train_loss": 0.002405446022748947, "cos_r_negg": [0.24503980576992035, 0.28094232082366943, 0.3298783302307129], "cos_innovation_negg": [0.24503980576992035, 0.28094232082366943, 0.3298783302307129], "cos_apical_negg": [0.00806405022740364, 0.03380434215068817, 0.08361522108316422], "cos_Ac_negg": [0.5700715184211731, 0.6392761468887329, 0.6756144762039185], "r_norm": [0.13945350050926208, 0.13656724989414215, 0.1388821005821228]}, {"step": 6000, "epoch": 12, "train_loss": 0.00017605547327548265, "cos_r_negg": [0.23001554608345032, 0.2646538019180298, 0.31029847264289856], "cos_innovation_negg": [0.23001554608345032, 0.2646538019180298, 0.31029847264289856], "cos_apical_negg": [0.00541045144200325, 0.032030005007982254, 0.07905453443527222], "cos_Ac_negg": [0.5708320140838623, 0.6355617046356201, 0.6697124242782593], "r_norm": [0.13937880098819733, 0.13642413914203644, 0.13876959681510925], "ldr": {"dL_sdil": -0.0074355751276016235, "dL_bp": -0.012880563735961914, "ratio": 0.5772709393799167}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9744, "test_loss": 0.13820682907104492}, {"step": 6100, "epoch": 13, "train_loss": 0.0009763767011463642, "cos_r_negg": [0.23343919217586517, 0.2614392042160034, 0.3084016740322113], "cos_innovation_negg": [0.23343919217586517, 0.2614392042160034, 0.3084016740322113], "cos_apical_negg": [0.006905930116772652, 0.02927277237176895, 0.07730467617511749], "cos_Ac_negg": [0.5668554902076721, 0.6321935653686523, 0.6682334542274475], "r_norm": [0.1409309208393097, 0.1382412612438202, 0.14066889882087708]}, {"step": 6200, "epoch": 13, "train_loss": 0.0009069643565453589, "cos_r_negg": [0.22708435356616974, 0.25632721185684204, 0.3010939657688141], "cos_innovation_negg": [0.22708435356616974, 0.25632721185684204, 0.3010939657688141], "cos_apical_negg": [0.008822782896459103, 0.029689308255910873, 0.07625748962163925], "cos_Ac_negg": [0.5691685676574707, 0.6327530145645142, 0.6673669815063477], "r_norm": [0.1352216601371765, 0.13269278407096863, 0.13488000631332397]}, {"step": 6300, "epoch": 13, "train_loss": 0.0006839922862127423, "cos_r_negg": [0.21666383743286133, 0.24478955566883087, 0.2880740165710449], "cos_innovation_negg": [0.21666383743286133, 0.24478955566883087, 0.2880740165710449], "cos_apical_negg": [0.006035208702087402, 0.029654499143362045, 0.07456348836421967], "cos_Ac_negg": [0.5658756494522095, 0.6283625364303589, 0.6640976667404175], "r_norm": [0.1393083930015564, 0.136541485786438, 0.13898012042045593]}, {"step": 6400, "epoch": 13, "train_loss": 0.0005505380686372519, "cos_r_negg": [0.21647170186042786, 0.24702751636505127, 0.29178813099861145], "cos_innovation_negg": [0.21647170186042786, 0.24702751636505127, 0.29178813099861145], "cos_apical_negg": [0.006189728155732155, 0.02983001060783863, 0.0744553953409195], "cos_Ac_negg": [0.5681161284446716, 0.6294869184494019, 0.6676573157310486], "r_norm": [0.14122754335403442, 0.13862696290016174, 0.14077913761138916]}, {"step": 6500, "epoch": 13, "train_loss": 0.00035615594242699444, "cos_r_negg": [0.21158862113952637, 0.2439221888780594, 0.28754645586013794], "cos_innovation_negg": [0.21158862113952637, 0.2439221888780594, 0.28754645586013794], "cos_apical_negg": [0.0036592865362763405, 0.029765024781227112, 0.0722854882478714], "cos_Ac_negg": [0.5633366107940674, 0.6278918981552124, 0.6642735600471497], "r_norm": [0.1363537609577179, 0.13358211517333984, 0.13598135113716125], "ldr": {"dL_sdil": -0.007500775158405304, "dL_bp": -0.013501659035682678, "ratio": 0.5555447029570204}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9739, "test_loss": 0.13985029439926147}, {"step": 6600, "epoch": 14, "train_loss": 0.0006743880803696811, "cos_r_negg": [0.2092241793870926, 0.24195091426372528, 0.28283411264419556], "cos_innovation_negg": [0.2092241793870926, 0.24195091426372528, 0.28283411264419556], "cos_apical_negg": [0.006467920728027821, 0.02924356423318386, 0.07268835604190826], "cos_Ac_negg": [0.5627449750900269, 0.6298940181732178, 0.6678542494773865], "r_norm": [0.13558074831962585, 0.13304200768470764, 0.1353493332862854]}, {"step": 6700, "epoch": 14, "train_loss": 0.00031959370244294405, "cos_r_negg": [0.20629185438156128, 0.23710137605667114, 0.2809441089630127], "cos_innovation_negg": [0.20629185438156128, 0.23710137605667114, 0.2809441089630127], "cos_apical_negg": [0.0054047489538788795, 0.029138857498764992, 0.07521852850914001], "cos_Ac_negg": [0.5613837242126465, 0.6291850805282593, 0.6665549278259277], "r_norm": [0.13533002138137817, 0.13286085426807404, 0.13514675199985504]}, {"step": 6800, "epoch": 14, "train_loss": 0.0004982365644536912, "cos_r_negg": [0.19535169005393982, 0.2248317003250122, 0.2654002904891968], "cos_innovation_negg": [0.19535169005393982, 0.2248317003250122, 0.2654002904891968], "cos_apical_negg": [0.005468869116157293, 0.028167756274342537, 0.07059791684150696], "cos_Ac_negg": [0.5609909296035767, 0.6257888078689575, 0.6647816896438599], "r_norm": [0.13461200892925262, 0.1320335566997528, 0.13426797091960907]}, {"step": 6900, "epoch": 14, "train_loss": 0.0008965390734374523, "cos_r_negg": [0.19515705108642578, 0.22602930665016174, 0.26685693860054016], "cos_innovation_negg": [0.19515705108642578, 0.22602930665016174, 0.26685693860054016], "cos_apical_negg": [0.00480653764680028, 0.0283636637032032, 0.07082416117191315], "cos_Ac_negg": [0.5626923441886902, 0.6275756359100342, 0.6644430160522461], "r_norm": [0.13189470767974854, 0.129350483417511, 0.13152959942817688]}, {"step": 7000, "epoch": 14, "train_loss": 0.00037491926923394203, "cos_r_negg": [0.19524270296096802, 0.2249695062637329, 0.26578110456466675], "cos_innovation_negg": [0.19524270296096802, 0.2249695062637329, 0.26578110456466675], "cos_apical_negg": [0.004228589590638876, 0.02768980711698532, 0.07010260224342346], "cos_Ac_negg": [0.56703782081604, 0.6305042505264282, 0.6680166721343994], "r_norm": [0.1324881613254547, 0.13006103038787842, 0.13218775391578674], "ldr": {"dL_sdil": -0.007371082901954651, "dL_bp": -0.013634532690048218, "ratio": 0.5406186680189465}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9741, "test_loss": 0.14238088626861573}], "final": {"test_acc": 0.9741, "test_loss": 0.14238088626861573, "wall_s": 23.89514470100403, "cos_r_negg": [0.19353516399860382, 0.2238001823425293, 0.2641659080982208], "cos_innovation_negg": [0.19353516399860382, 0.2238001823425293, 0.2641659080982208], "cos_apical_negg": [0.004093382507562637, 0.02720249630510807, 0.06915740668773651], "cos_Ac_negg": [0.5639503002166748, 0.6269025802612305, 0.6651663184165955]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p2_residual_s4.json b/results/nuis_ctrl_v1_rho0p2_residual_s4.json new file mode 100644 index 0000000..612a1c7 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p2_residual_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.2, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p2_residual_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.012779776938259602, 0.029937468469142914, 0.009443523362278938], "cos_innovation_negg": [-0.012779776938259602, 0.029937468469142914, 0.009443523362278938], "cos_apical_negg": [-0.014390179887413979, 0.02703464776277542, 0.006248843390494585], "cos_Ac_negg": [-0.012780384160578251, 0.029937095940113068, 0.009443294256925583], "r_norm": [4.848398208618164, 4.8553009033203125, 4.948000907897949], "ldr": {"dL_sdil": -0.003349781036376953, "dL_bp": -0.26721620559692383, "ratio": 0.012535845379938721}}, {"step": 100, "epoch": 0, "train_loss": 0.19523030519485474, "cos_r_negg": [0.45847442746162415, 0.37912869453430176, 0.3189619779586792], "cos_innovation_negg": [0.45847442746162415, 0.37912869453430176, 0.3189619779586792], "cos_apical_negg": [0.11247933655977249, 0.13046348094940186, 0.1326301544904709], "cos_Ac_negg": [0.4592515826225281, 0.3796048164367676, 0.31860604882240295], "r_norm": [0.9187071919441223, 0.9284980297088623, 0.943510890007019]}, {"step": 200, "epoch": 0, "train_loss": 0.24339696764945984, "cos_r_negg": [0.5118288993835449, 0.44126784801483154, 0.3616797924041748], "cos_innovation_negg": [0.5118288993835449, 0.44126784801483154, 0.3616797924041748], "cos_apical_negg": [0.12309147417545319, 0.13048416376113892, 0.14339151978492737], "cos_Ac_negg": [0.5134156346321106, 0.4426364302635193, 0.360500693321228], "r_norm": [0.7761707901954651, 0.7813184857368469, 0.7995097041130066]}, {"step": 300, "epoch": 0, "train_loss": 0.25752902030944824, "cos_r_negg": [0.528708815574646, 0.4626166522502899, 0.37011367082595825], "cos_innovation_negg": [0.528708815574646, 0.4626166522502899, 0.37011367082595825], "cos_apical_negg": [0.10177868604660034, 0.12452653795480728, 0.126957505941391], "cos_Ac_negg": [0.5307761430740356, 0.46267470717430115, 0.36981648206710815], "r_norm": [0.5919914245605469, 0.5966252684593201, 0.6120467185974121]}, {"step": 400, "epoch": 0, "train_loss": 0.13028958439826965, "cos_r_negg": [0.5409702062606812, 0.4840485453605652, 0.42399299144744873], "cos_innovation_negg": [0.5409702062606812, 0.4840485453605652, 0.42399299144744873], "cos_apical_negg": [0.11041324585676193, 0.11936397850513458, 0.12784609198570251], "cos_Ac_negg": [0.5410827994346619, 0.4839455187320709, 0.4234982132911682], "r_norm": [0.5470607876777649, 0.5520876049995422, 0.5642469525337219]}, {"epoch_end": 0, "step": 469, "test_acc": 0.947, "test_loss": 0.17841850128173828}, {"step": 500, "epoch": 1, "train_loss": 0.1807648241519928, "cos_r_negg": [0.5348552465438843, 0.5115889310836792, 0.4514876902103424], "cos_innovation_negg": [0.5348552465438843, 0.5115889310836792, 0.4514876902103424], "cos_apical_negg": [0.0851651281118393, 0.11966394633054733, 0.14762669801712036], "cos_Ac_negg": [0.5365169644355774, 0.5125844478607178, 0.4526808261871338], "r_norm": [0.5111985206604004, 0.5184361934661865, 0.5298259258270264], "ldr": {"dL_sdil": -0.01582382619380951, "dL_bp": -0.013664066791534424, "ratio": 1.1580612445200549}}, {"step": 600, "epoch": 1, "train_loss": 0.07896888256072998, "cos_r_negg": [0.5404101610183716, 0.5252118706703186, 0.42411017417907715], "cos_innovation_negg": [0.5404101610183716, 0.5252118706703186, 0.42411017417907715], "cos_apical_negg": [0.07069253921508789, 0.10805739462375641, 0.1227489709854126], "cos_Ac_negg": [0.5426425933837891, 0.5250234007835388, 0.42436668276786804], "r_norm": [0.4426986277103424, 0.4445212185382843, 0.45536065101623535]}, {"step": 700, "epoch": 1, "train_loss": 0.1936294138431549, "cos_r_negg": [0.5523302555084229, 0.5313393473625183, 0.4363704025745392], "cos_innovation_negg": [0.5523302555084229, 0.5313393473625183, 0.4363704025745392], "cos_apical_negg": [0.07309302687644958, 0.12037917226552963, 0.12745285034179688], "cos_Ac_negg": [0.5567302703857422, 0.5333513617515564, 0.439350962638855], "r_norm": [0.47314751148223877, 0.47849762439727783, 0.4916081428527832]}, {"step": 800, "epoch": 1, "train_loss": 0.17381304502487183, "cos_r_negg": [0.5510797500610352, 0.5477213859558105, 0.4589617848396301], "cos_innovation_negg": [0.5510797500610352, 0.5477213859558105, 0.4589617848396301], "cos_apical_negg": [0.0753069743514061, 0.10238130390644073, 0.13298794627189636], "cos_Ac_negg": [0.5540711283683777, 0.5492314100265503, 0.4595499634742737], "r_norm": [0.36570852994918823, 0.3675145208835602, 0.3761633038520813]}, {"step": 900, "epoch": 1, "train_loss": 0.2041294127702713, "cos_r_negg": [0.5688360929489136, 0.5709272623062134, 0.4859267473220825], "cos_innovation_negg": [0.5688360929489136, 0.5709272623062134, 0.4859267473220825], "cos_apical_negg": [0.07023810595273972, 0.12307806313037872, 0.14232924580574036], "cos_Ac_negg": [0.5693740844726562, 0.5711548328399658, 0.48617517948150635], "r_norm": [0.47735312581062317, 0.4816506505012512, 0.48987314105033875]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9522, "test_loss": 0.164680269241333}, {"step": 1000, "epoch": 2, "train_loss": 0.03263162076473236, "cos_r_negg": [0.5729007720947266, 0.5703027248382568, 0.4814510941505432], "cos_innovation_negg": [0.5729007720947266, 0.5703027248382568, 0.4814510941505432], "cos_apical_negg": [0.05268578976392746, 0.09828369319438934, 0.12499398738145828], "cos_Ac_negg": [0.5749251842498779, 0.5714312791824341, 0.4817565381526947], "r_norm": [0.3504492938518524, 0.3510257601737976, 0.3581618666648865], "ldr": {"dL_sdil": -0.008551239967346191, "dL_bp": -0.0071846842765808105, "ratio": 1.1902040003650272}}, {"step": 1100, "epoch": 2, "train_loss": 0.13308821618556976, "cos_r_negg": [0.5845746994018555, 0.5814930200576782, 0.48799845576286316], "cos_innovation_negg": [0.5845746994018555, 0.5814930200576782, 0.48799845576286316], "cos_apical_negg": [0.057091135531663895, 0.10458144545555115, 0.14837689697742462], "cos_Ac_negg": [0.5915025472640991, 0.5857481956481934, 0.48922598361968994], "r_norm": [0.38480323553085327, 0.38755786418914795, 0.39720049500465393]}, {"step": 1200, "epoch": 2, "train_loss": 0.1610100418329239, "cos_r_negg": [0.5727114677429199, 0.5879212617874146, 0.49369505047798157], "cos_innovation_negg": [0.5727114677429199, 0.5879212617874146, 0.49369505047798157], "cos_apical_negg": [0.03205304965376854, 0.0988479033112526, 0.1303451955318451], "cos_Ac_negg": [0.5806068181991577, 0.5938493013381958, 0.49549737572669983], "r_norm": [0.316397100687027, 0.31673741340637207, 0.32411086559295654]}, {"step": 1300, "epoch": 2, "train_loss": 0.12316633760929108, "cos_r_negg": [0.5860399007797241, 0.5971480011940002, 0.48872262239456177], "cos_innovation_negg": [0.5860399007797241, 0.5971480011940002, 0.48872262239456177], "cos_apical_negg": [0.043977562338113785, 0.09919430315494537, 0.13200163841247559], "cos_Ac_negg": [0.5901158452033997, 0.5985755324363708, 0.48968470096588135], "r_norm": [0.31413739919662476, 0.3152046799659729, 0.32350340485572815]}, {"step": 1400, "epoch": 2, "train_loss": 0.13651013374328613, "cos_r_negg": [0.5844224691390991, 0.5855025053024292, 0.5037516951560974], "cos_innovation_negg": [0.5844224691390991, 0.5855025053024292, 0.5037516951560974], "cos_apical_negg": [0.05487751215696335, 0.09212378412485123, 0.12417959421873093], "cos_Ac_negg": [0.5932061672210693, 0.5950719118118286, 0.5087454319000244], "r_norm": [0.30646589398384094, 0.30732524394989014, 0.315845251083374]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9648, "test_loss": 0.1194626262664795}, {"step": 1500, "epoch": 3, "train_loss": 0.07648062705993652, "cos_r_negg": [0.582809329032898, 0.5990599393844604, 0.5306364297866821], "cos_innovation_negg": [0.582809329032898, 0.5990599393844604, 0.5306364297866821], "cos_apical_negg": [0.05203244462609291, 0.07996759563684464, 0.1280796378850937], "cos_Ac_negg": [0.5888310074806213, 0.6044595241546631, 0.5320906639099121], "r_norm": [0.26579931378364563, 0.2674559950828552, 0.2740623354911804], "ldr": {"dL_sdil": -0.006355859339237213, "dL_bp": -0.005833715200424194, "ratio": 1.089504564565485}}, {"step": 1600, "epoch": 3, "train_loss": 0.0905633419752121, "cos_r_negg": [0.5723593235015869, 0.5879327058792114, 0.5362997055053711], "cos_innovation_negg": [0.5723593235015869, 0.5879327058792114, 0.5362997055053711], "cos_apical_negg": [0.04090467840433121, 0.09057846665382385, 0.13063514232635498], "cos_Ac_negg": [0.582373857498169, 0.599800705909729, 0.5396597385406494], "r_norm": [0.3216865658760071, 0.32372450828552246, 0.32970255613327026]}, {"step": 1700, "epoch": 3, "train_loss": 0.03675437346100807, "cos_r_negg": [0.5639013648033142, 0.5984659790992737, 0.5404385328292847], "cos_innovation_negg": [0.5639013648033142, 0.5984659790992737, 0.5404385328292847], "cos_apical_negg": [0.0356726348400116, 0.08682107925415039, 0.12229768186807632], "cos_Ac_negg": [0.5738296508789062, 0.6086991429328918, 0.5423985719680786], "r_norm": [0.3025714159011841, 0.302367240190506, 0.30920514464378357]}, {"step": 1800, "epoch": 3, "train_loss": 0.012730604037642479, "cos_r_negg": [0.5734977722167969, 0.5982744693756104, 0.5369141101837158], "cos_innovation_negg": [0.5734977722167969, 0.5982744693756104, 0.5369141101837158], "cos_apical_negg": [0.03080776333808899, 0.08617154508829117, 0.11317454278469086], "cos_Ac_negg": [0.586790144443512, 0.6150208711624146, 0.5431820154190063], "r_norm": [0.2756425440311432, 0.27534306049346924, 0.2815254330635071]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9663, "test_loss": 0.11451079864501953}, {"step": 1900, "epoch": 4, "train_loss": 0.032762207090854645, "cos_r_negg": [0.5686196088790894, 0.6121808290481567, 0.5366095900535583], "cos_innovation_negg": [0.5686196088790894, 0.6121808290481567, 0.5366095900535583], "cos_apical_negg": [0.019442813470959663, 0.07997725903987885, 0.11869443953037262], "cos_Ac_negg": [0.5839101672172546, 0.6255109310150146, 0.5402666926383972], "r_norm": [0.2687967121601105, 0.26841211318969727, 0.2760213613510132]}, {"step": 2000, "epoch": 4, "train_loss": 0.07402737438678741, "cos_r_negg": [0.550966203212738, 0.5759673714637756, 0.5259472131729126], "cos_innovation_negg": [0.550966203212738, 0.5759673714637756, 0.5259472131729126], "cos_apical_negg": [0.03513997420668602, 0.08047322928905487, 0.12142512202262878], "cos_Ac_negg": [0.5936437845230103, 0.6221067905426025, 0.5487080812454224], "r_norm": [0.2583365738391876, 0.2562132775783539, 0.2620803713798523], "ldr": {"dL_sdil": -0.008371703326702118, "dL_bp": -0.00857352465391159, "ratio": 0.976459935049304}}, {"step": 2100, "epoch": 4, "train_loss": 0.042866051197052, "cos_r_negg": [0.5705348253250122, 0.599862813949585, 0.5541979074478149], "cos_innovation_negg": [0.5705348253250122, 0.599862813949585, 0.5541979074478149], "cos_apical_negg": [0.05051989108324051, 0.08365020155906677, 0.13147062063217163], "cos_Ac_negg": [0.5924770832061768, 0.6245070695877075, 0.5610617399215698], "r_norm": [0.28235897421836853, 0.2823737859725952, 0.2904585003852844]}, {"step": 2200, "epoch": 4, "train_loss": 0.010305065661668777, "cos_r_negg": [0.5772885680198669, 0.607243537902832, 0.5564123392105103], "cos_innovation_negg": [0.5772885680198669, 0.607243537902832, 0.5564123392105103], "cos_apical_negg": [0.03302644193172455, 0.07795511931180954, 0.1127902939915657], "cos_Ac_negg": [0.6015622615814209, 0.632867693901062, 0.5630269646644592], "r_norm": [0.25257766246795654, 0.25193700194358826, 0.25840598344802856]}, {"step": 2300, "epoch": 4, "train_loss": 0.15979208052158356, "cos_r_negg": [0.5494014024734497, 0.5776619911193848, 0.5485407710075378], "cos_innovation_negg": [0.5494014024734497, 0.5776619911193848, 0.5485407710075378], "cos_apical_negg": [0.03557701036334038, 0.07048080116510391, 0.11928121745586395], "cos_Ac_negg": [0.5928793549537659, 0.6328431367874146, 0.5666439533233643], "r_norm": [0.25622111558914185, 0.25602421164512634, 0.26393648982048035]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9707, "test_loss": 0.11497614192962646}, {"step": 2400, "epoch": 5, "train_loss": 0.1078932136297226, "cos_r_negg": [0.5551625490188599, 0.5934861898422241, 0.5524869561195374], "cos_innovation_negg": [0.5551625490188599, 0.5934861898422241, 0.5524869561195374], "cos_apical_negg": [0.028840474784374237, 0.06992577016353607, 0.11109043657779694], "cos_Ac_negg": [0.6028305292129517, 0.6434247493743896, 0.5699541568756104], "r_norm": [0.21299418807029724, 0.21166028082370758, 0.21788239479064941]}, {"step": 2500, "epoch": 5, "train_loss": 0.02374871075153351, "cos_r_negg": [0.535438060760498, 0.5594796538352966, 0.5295423269271851], "cos_innovation_negg": [0.535438060760498, 0.5594796538352966, 0.5295423269271851], "cos_apical_negg": [0.023892957717180252, 0.06995686888694763, 0.12139956653118134], "cos_Ac_negg": [0.6035381555557251, 0.6296474933624268, 0.5638284087181091], "r_norm": [0.2603135108947754, 0.2593501806259155, 0.26533472537994385], "ldr": {"dL_sdil": -0.009871810674667358, "dL_bp": -0.011774621903896332, "ratio": 0.8383972543017015}}, {"step": 2600, "epoch": 5, "train_loss": 0.05304960906505585, "cos_r_negg": [0.52979576587677, 0.5670466423034668, 0.5479004979133606], "cos_innovation_negg": [0.52979576587677, 0.5670466423034668, 0.5479004979133606], "cos_apical_negg": [0.04220541939139366, 0.08032345771789551, 0.1262071281671524], "cos_Ac_negg": [0.6043375730514526, 0.6363017559051514, 0.58698570728302], "r_norm": [0.22474563121795654, 0.22321724891662598, 0.23007771372795105]}, {"step": 2700, "epoch": 5, "train_loss": 0.034139081835746765, "cos_r_negg": [0.529230535030365, 0.5677273273468018, 0.539835512638092], "cos_innovation_negg": [0.529230535030365, 0.5677273273468018, 0.539835512638092], "cos_apical_negg": [0.024656590074300766, 0.07033709436655045, 0.11462977528572083], "cos_Ac_negg": [0.6117772459983826, 0.6435583829879761, 0.5806998014450073], "r_norm": [0.2135850340127945, 0.21193356812000275, 0.21844452619552612]}, {"step": 2800, "epoch": 5, "train_loss": 0.034261301159858704, "cos_r_negg": [0.5269429683685303, 0.5670950412750244, 0.5415679216384888], "cos_innovation_negg": [0.5269429683685303, 0.5670950412750244, 0.5415679216384888], "cos_apical_negg": [0.021020326763391495, 0.07434876263141632, 0.11197115480899811], "cos_Ac_negg": [0.6107158660888672, 0.642261266708374, 0.5806410312652588], "r_norm": [0.2211204618215561, 0.21918120980262756, 0.2257131189107895]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.97, "test_loss": 0.11780296745300294}, {"step": 2900, "epoch": 6, "train_loss": 0.011397719383239746, "cos_r_negg": [0.5219845771789551, 0.5616589784622192, 0.542546808719635], "cos_innovation_negg": [0.5219845771789551, 0.5616589784622192, 0.542546808719635], "cos_apical_negg": [0.02675338089466095, 0.07400087267160416, 0.11950907111167908], "cos_Ac_negg": [0.6122092008590698, 0.6477441787719727, 0.5883749723434448], "r_norm": [0.23567363619804382, 0.2354898452758789, 0.24229687452316284]}, {"step": 3000, "epoch": 6, "train_loss": 0.06766141206026077, "cos_r_negg": [0.5165479183197021, 0.5659703612327576, 0.5425368547439575], "cos_innovation_negg": [0.5165479183197021, 0.5659703612327576, 0.5425368547439575], "cos_apical_negg": [0.018119461834430695, 0.07276603579521179, 0.11712069064378738], "cos_Ac_negg": [0.6148598194122314, 0.6590943932533264, 0.5922616720199585], "r_norm": [0.2285493016242981, 0.2258799523115158, 0.23218761384487152], "ldr": {"dL_sdil": -0.008624427020549774, "dL_bp": -0.011231310665607452, "ratio": 0.7678914133289462}}, {"step": 3100, "epoch": 6, "train_loss": 0.011940602213144302, "cos_r_negg": [0.49066847562789917, 0.5448637008666992, 0.5339353084564209], "cos_innovation_negg": [0.49066847562789917, 0.5448637008666992, 0.5339353084564209], "cos_apical_negg": [0.007285827770829201, 0.06124396622180939, 0.11916914582252502], "cos_Ac_negg": [0.6013911962509155, 0.6561592817306519, 0.5947411060333252], "r_norm": [0.175805002450943, 0.17356181144714355, 0.17831215262413025]}, {"step": 3200, "epoch": 6, "train_loss": 0.01863642781972885, "cos_r_negg": [0.4804835319519043, 0.5327173471450806, 0.5243409872055054], "cos_innovation_negg": [0.4804835319519043, 0.5327173471450806, 0.5243409872055054], "cos_apical_negg": [0.013016757555305958, 0.061587098985910416, 0.1157207190990448], "cos_Ac_negg": [0.6106147170066833, 0.6626662015914917, 0.5968565344810486], "r_norm": [0.1887008249759674, 0.18547572195529938, 0.19113068282604218]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9738, "test_loss": 0.11375721931457519}, {"step": 3300, "epoch": 7, "train_loss": 0.010094470344483852, "cos_r_negg": [0.4903387427330017, 0.5452684760093689, 0.5329725742340088], "cos_innovation_negg": [0.4903387427330017, 0.5452684760093689, 0.5329725742340088], "cos_apical_negg": [0.023183079436421394, 0.06250479817390442, 0.1161399632692337], "cos_Ac_negg": [0.609476625919342, 0.662117600440979, 0.600680947303772], "r_norm": [0.1918632984161377, 0.18966834247112274, 0.19530579447746277]}, {"step": 3400, "epoch": 7, "train_loss": 0.028262287378311157, "cos_r_negg": [0.4746207594871521, 0.5208358764648438, 0.5210437178611755], "cos_innovation_negg": [0.4746207594871521, 0.5208358764648438, 0.5210437178611755], "cos_apical_negg": [0.030959200114011765, 0.064633809030056, 0.11897359043359756], "cos_Ac_negg": [0.6199973821640015, 0.6660165786743164, 0.6130518913269043], "r_norm": [0.237336665391922, 0.23466970026493073, 0.2413531392812729]}, {"step": 3500, "epoch": 7, "train_loss": 0.012605790048837662, "cos_r_negg": [0.4489513337612152, 0.5001634359359741, 0.5048023462295532], "cos_innovation_negg": [0.4489513337612152, 0.5001634359359741, 0.5048023462295532], "cos_apical_negg": [0.011033246293663979, 0.06124911829829216, 0.11606670916080475], "cos_Ac_negg": [0.6154003739356995, 0.664509654045105, 0.619300365447998], "r_norm": [0.18742799758911133, 0.18515513837337494, 0.1903511881828308], "ldr": {"dL_sdil": -0.00667426735162735, "dL_bp": -0.009379982948303223, "ratio": 0.7115436550803839}}, {"step": 3600, "epoch": 7, "train_loss": 0.002501162001863122, "cos_r_negg": [0.4633944630622864, 0.5094193816184998, 0.5131481885910034], "cos_innovation_negg": [0.4633944630622864, 0.5094193816184998, 0.5131481885910034], "cos_apical_negg": [0.014765365980565548, 0.06386268138885498, 0.11611031740903854], "cos_Ac_negg": [0.6221897602081299, 0.6654284000396729, 0.6188482046127319], "r_norm": [0.19960951805114746, 0.19723404943943024, 0.20290613174438477]}, {"step": 3700, "epoch": 7, "train_loss": 0.047592177987098694, "cos_r_negg": [0.4487311840057373, 0.4822712242603302, 0.48509272933006287], "cos_innovation_negg": [0.4487311840057373, 0.4822712242603302, 0.48509272933006287], "cos_apical_negg": [0.016713546589016914, 0.06394006311893463, 0.10898618400096893], "cos_Ac_negg": [0.6234561204910278, 0.6643760204315186, 0.6124838590621948], "r_norm": [0.22732378542423248, 0.224981427192688, 0.23132050037384033]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9739, "test_loss": 0.1135302433013916}, {"step": 3800, "epoch": 8, "train_loss": 0.003513964358717203, "cos_r_negg": [0.42559632658958435, 0.46762120723724365, 0.46973276138305664], "cos_innovation_negg": [0.42559632658958435, 0.46762120723724365, 0.46973276138305664], "cos_apical_negg": [0.01379104983061552, 0.06488192081451416, 0.10912670195102692], "cos_Ac_negg": [0.6189025044441223, 0.6681501865386963, 0.6110430955886841], "r_norm": [0.19996729493141174, 0.1962798684835434, 0.20215703547000885]}, {"step": 3900, "epoch": 8, "train_loss": 0.017354629933834076, "cos_r_negg": [0.3996793031692505, 0.45104163885116577, 0.4575529396533966], "cos_innovation_negg": [0.3996793031692505, 0.45104163885116577, 0.4575529396533966], "cos_apical_negg": [0.01126004196703434, 0.0607152059674263, 0.1086505800485611], "cos_Ac_negg": [0.6127740740776062, 0.6692689061164856, 0.6164525747299194], "r_norm": [0.1811152696609497, 0.17744076251983643, 0.18241417407989502]}, {"step": 4000, "epoch": 8, "train_loss": 0.004856348503381014, "cos_r_negg": [0.37602561712265015, 0.41966673731803894, 0.43439748883247375], "cos_innovation_negg": [0.37602561712265015, 0.41966673731803894, 0.43439748883247375], "cos_apical_negg": [0.008020796813070774, 0.057990677654743195, 0.09969477355480194], "cos_Ac_negg": [0.6132957339286804, 0.6661556959152222, 0.6181684136390686], "r_norm": [0.16387838125228882, 0.16086477041244507, 0.16547046601772308], "ldr": {"dL_sdil": -0.007244512438774109, "dL_bp": -0.011032260954380035, "ratio": 0.6566661601580307}}, {"step": 4100, "epoch": 8, "train_loss": 0.013123110868036747, "cos_r_negg": [0.4056463837623596, 0.45457643270492554, 0.4590320885181427], "cos_innovation_negg": [0.4056463837623596, 0.45457643270492554, 0.4590320885181427], "cos_apical_negg": [0.011253751814365387, 0.06664575636386871, 0.10164335370063782], "cos_Ac_negg": [0.6166028380393982, 0.6658480167388916, 0.621646523475647], "r_norm": [0.20974087715148926, 0.2073712944984436, 0.21274694800376892]}, {"step": 4200, "epoch": 8, "train_loss": 0.020602025091648102, "cos_r_negg": [0.37572383880615234, 0.4216482937335968, 0.44565603137016296], "cos_innovation_negg": [0.37572383880615234, 0.4216482937335968, 0.44565603137016296], "cos_apical_negg": [0.003575623035430908, 0.05617605894804001, 0.10992729663848877], "cos_Ac_negg": [0.6122173070907593, 0.6653400659561157, 0.627249002456665], "r_norm": [0.2008437216281891, 0.1983548402786255, 0.2032732367515564]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9739, "test_loss": 0.12465989742279053}, {"step": 4300, "epoch": 9, "train_loss": 0.005610136780887842, "cos_r_negg": [0.3538643717765808, 0.3970605731010437, 0.42124608159065247], "cos_innovation_negg": [0.3538643717765808, 0.3970605731010437, 0.42124608159065247], "cos_apical_negg": [0.007544706575572491, 0.056428320705890656, 0.11097771674394608], "cos_Ac_negg": [0.619644045829773, 0.6742088794708252, 0.6325809955596924], "r_norm": [0.20761768519878387, 0.20515799522399902, 0.20977666974067688]}, {"step": 4400, "epoch": 9, "train_loss": 0.005376679822802544, "cos_r_negg": [0.3503841161727905, 0.3912612795829773, 0.40973806381225586], "cos_innovation_negg": [0.3503841161727905, 0.3912612795829773, 0.40973806381225586], "cos_apical_negg": [0.009758060798048973, 0.05871321260929108, 0.10685720294713974], "cos_Ac_negg": [0.6169899106025696, 0.6703310012817383, 0.6331541538238525], "r_norm": [0.20245492458343506, 0.19962534308433533, 0.2046738564968109]}, {"step": 4500, "epoch": 9, "train_loss": 0.036763302981853485, "cos_r_negg": [0.33551520109176636, 0.37393367290496826, 0.3952620327472687], "cos_innovation_negg": [0.33551520109176636, 0.37393367290496826, 0.3952620327472687], "cos_apical_negg": [0.007033657282590866, 0.0598057396709919, 0.10435810685157776], "cos_Ac_negg": [0.619627058506012, 0.667587161064148, 0.6292416453361511], "r_norm": [0.20359715819358826, 0.201330304145813, 0.20609556138515472], "ldr": {"dL_sdil": -0.010631203651428223, "dL_bp": -0.01770325005054474, "ratio": 0.6005227074732018}}, {"step": 4600, "epoch": 9, "train_loss": 0.0037832045927643776, "cos_r_negg": [0.3121258616447449, 0.36032769083976746, 0.3776133954524994], "cos_innovation_negg": [0.3121258616447449, 0.36032769083976746, 0.3776133954524994], "cos_apical_negg": [0.015989728271961212, 0.06058131158351898, 0.10416124761104584], "cos_Ac_negg": [0.6033607721328735, 0.6596585512161255, 0.623542070388794], "r_norm": [0.2018270641565323, 0.19944502413272858, 0.2047208696603775]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9753, "test_loss": 0.12550905532836915}, {"step": 4700, "epoch": 10, "train_loss": 0.003072786144912243, "cos_r_negg": [0.2944188416004181, 0.32908353209495544, 0.35645240545272827], "cos_innovation_negg": [0.2944188416004181, 0.32908353209495544, 0.35645240545272827], "cos_apical_negg": [0.016930514946579933, 0.05618127062916756, 0.10497274249792099], "cos_Ac_negg": [0.6030189990997314, 0.655601441860199, 0.6225085854530334], "r_norm": [0.20414391160011292, 0.2015935778617859, 0.2070946842432022]}, {"step": 4800, "epoch": 10, "train_loss": 0.0008679613820277154, "cos_r_negg": [0.2837217450141907, 0.3221810460090637, 0.34900766611099243], "cos_innovation_negg": [0.2837217450141907, 0.3221810460090637, 0.34900766611099243], "cos_apical_negg": [0.015292130410671234, 0.05809130147099495, 0.10039294511079788], "cos_Ac_negg": [0.6085541844367981, 0.6584851145744324, 0.6256493330001831], "r_norm": [0.20482318103313446, 0.20216301083564758, 0.20747578144073486]}, {"step": 4900, "epoch": 10, "train_loss": 0.000689456588588655, "cos_r_negg": [0.29323333501815796, 0.33389848470687866, 0.3601685166358948], "cos_innovation_negg": [0.29323333501815796, 0.33389848470687866, 0.3601685166358948], "cos_apical_negg": [0.013220809400081635, 0.0575682707130909, 0.10602718591690063], "cos_Ac_negg": [0.6104445457458496, 0.6623284816741943, 0.6290529370307922], "r_norm": [0.20441797375679016, 0.2013956904411316, 0.20674476027488708]}, {"step": 5000, "epoch": 10, "train_loss": 0.002489793812856078, "cos_r_negg": [0.271761029958725, 0.3122059106826782, 0.33744847774505615], "cos_innovation_negg": [0.271761029958725, 0.3122059106826782, 0.33744847774505615], "cos_apical_negg": [0.004920518957078457, 0.052379921078681946, 0.09670887887477875], "cos_Ac_negg": [0.6035255789756775, 0.6595746278762817, 0.6313233971595764], "r_norm": [0.1856590211391449, 0.1828835904598236, 0.18777750432491302], "ldr": {"dL_sdil": -0.00991879403591156, "dL_bp": -0.017432942986488342, "ratio": 0.5689684205127766}}, {"step": 5100, "epoch": 10, "train_loss": 0.004342309199273586, "cos_r_negg": [0.2780485153198242, 0.3129868805408478, 0.3403642177581787], "cos_innovation_negg": [0.2780485153198242, 0.3129868805408478, 0.3403642177581787], "cos_apical_negg": [0.011281654238700867, 0.0573209710419178, 0.10063013434410095], "cos_Ac_negg": [0.606847882270813, 0.6650934219360352, 0.6345300078392029], "r_norm": [0.20424160361289978, 0.20170949399471283, 0.20710095763206482]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9749, "test_loss": 0.13234355907440185}, {"step": 5200, "epoch": 11, "train_loss": 0.0003710065211635083, "cos_r_negg": [0.26866614818573, 0.3031820058822632, 0.3304052948951721], "cos_innovation_negg": [0.26866614818573, 0.3031820058822632, 0.3304052948951721], "cos_apical_negg": [0.00830305926501751, 0.054413020610809326, 0.09989627450704575], "cos_Ac_negg": [0.6103471517562866, 0.6627001762390137, 0.6382420659065247], "r_norm": [0.19947873055934906, 0.19647422432899475, 0.20194187760353088]}, {"step": 5300, "epoch": 11, "train_loss": 0.001310388557612896, "cos_r_negg": [0.26046091318130493, 0.29670438170433044, 0.3230708837509155], "cos_innovation_negg": [0.26046091318130493, 0.29670438170433044, 0.3230708837509155], "cos_apical_negg": [0.003624703735113144, 0.0541350282728672, 0.09748238325119019], "cos_Ac_negg": [0.6059515476226807, 0.6595721244812012, 0.6382764577865601], "r_norm": [0.1994665414094925, 0.19642436504364014, 0.20184189081192017]}, {"step": 5400, "epoch": 11, "train_loss": 0.002470809267833829, "cos_r_negg": [0.25448858737945557, 0.2878149151802063, 0.3164098560810089], "cos_innovation_negg": [0.25448858737945557, 0.2878149151802063, 0.3164098560810089], "cos_apical_negg": [0.00452026491984725, 0.05260743573307991, 0.09517088532447815], "cos_Ac_negg": [0.6073060035705566, 0.6594626903533936, 0.6362428665161133], "r_norm": [0.1941535770893097, 0.19129878282546997, 0.1962924301624298]}, {"step": 5500, "epoch": 11, "train_loss": 0.002368900226429105, "cos_r_negg": [0.24936242401599884, 0.28135228157043457, 0.3080633878707886], "cos_innovation_negg": [0.24936242401599884, 0.28135228157043457, 0.3080633878707886], "cos_apical_negg": [0.00716947577893734, 0.051660820841789246, 0.091831274330616], "cos_Ac_negg": [0.6069409847259521, 0.6583317518234253, 0.6341017484664917], "r_norm": [0.1937277764081955, 0.1909867823123932, 0.19606642425060272], "ldr": {"dL_sdil": -0.01121484488248825, "dL_bp": -0.02140107750892639, "ratio": 0.5240317866149747}}, {"step": 5600, "epoch": 11, "train_loss": 0.0006278679938986897, "cos_r_negg": [0.24199867248535156, 0.2709590792655945, 0.2983156740665436], "cos_innovation_negg": [0.24199867248535156, 0.2709590792655945, 0.2983156740665436], "cos_apical_negg": [0.00971236452460289, 0.05224290490150452, 0.09043112397193909], "cos_Ac_negg": [0.6023350954055786, 0.6577886939048767, 0.6342024207115173], "r_norm": [0.19779030978679657, 0.1949300467967987, 0.20000602304935455]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9746, "test_loss": 0.13619753665924073}, {"step": 5700, "epoch": 12, "train_loss": 0.0006151688285171986, "cos_r_negg": [0.23634593188762665, 0.2623288035392761, 0.2931305766105652], "cos_innovation_negg": [0.23634593188762665, 0.2623288035392761, 0.2931305766105652], "cos_apical_negg": [0.006812026724219322, 0.04711467772722244, 0.08773602545261383], "cos_Ac_negg": [0.6036950945854187, 0.6565465927124023, 0.6355303525924683], "r_norm": [0.19501236081123352, 0.19225510954856873, 0.19715900719165802]}, {"step": 5800, "epoch": 12, "train_loss": 0.0005252254195511341, "cos_r_negg": [0.22883166372776031, 0.2535156011581421, 0.28534024953842163], "cos_innovation_negg": [0.22883166372776031, 0.2535156011581421, 0.28534024953842163], "cos_apical_negg": [0.00547691248357296, 0.04718494415283203, 0.08723905682563782], "cos_Ac_negg": [0.6058955192565918, 0.657025933265686, 0.6364992260932922], "r_norm": [0.1940699964761734, 0.19129586219787598, 0.1961495280265808]}, {"step": 5900, "epoch": 12, "train_loss": 0.001078218687325716, "cos_r_negg": [0.2313290238380432, 0.2567785084247589, 0.2869266867637634], "cos_innovation_negg": [0.2313290238380432, 0.2567785084247589, 0.2869266867637634], "cos_apical_negg": [0.005998366512358189, 0.04966701194643974, 0.0869307741522789], "cos_Ac_negg": [0.6020839214324951, 0.6555049419403076, 0.6366939544677734], "r_norm": [0.1965775191783905, 0.1937912553548813, 0.19868679344654083]}, {"step": 6000, "epoch": 12, "train_loss": 0.0009056039853021502, "cos_r_negg": [0.22886089980602264, 0.2550346851348877, 0.2855684757232666], "cos_innovation_negg": [0.22886089980602264, 0.2550346851348877, 0.2855684757232666], "cos_apical_negg": [0.0063621834851801395, 0.048350535333156586, 0.08751896023750305], "cos_Ac_negg": [0.6048064231872559, 0.6563602685928345, 0.637134313583374], "r_norm": [0.1951221227645874, 0.19225259125232697, 0.19705694913864136], "ldr": {"dL_sdil": -0.011640243232250214, "dL_bp": -0.023864418268203735, "ratio": 0.4877656392638466}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9752, "test_loss": 0.13830832386016845}, {"step": 6100, "epoch": 13, "train_loss": 0.00048515570233576, "cos_r_negg": [0.21801647543907166, 0.2452058345079422, 0.27572694420814514], "cos_innovation_negg": [0.21801647543907166, 0.2452058345079422, 0.27572694420814514], "cos_apical_negg": [0.004781823605298996, 0.047014400362968445, 0.0871681421995163], "cos_Ac_negg": [0.6030107140541077, 0.6560391187667847, 0.639000654220581], "r_norm": [0.19681480526924133, 0.19402167201042175, 0.199020117521286]}, {"step": 6200, "epoch": 13, "train_loss": 0.001061578281223774, "cos_r_negg": [0.21340787410736084, 0.23987334966659546, 0.2705043852329254], "cos_innovation_negg": [0.21340787410736084, 0.23987334966659546, 0.2705043852329254], "cos_apical_negg": [0.004939937964081764, 0.04700884968042374, 0.0865381509065628], "cos_Ac_negg": [0.603725790977478, 0.6578191518783569, 0.6400411128997803], "r_norm": [0.19550463557243347, 0.19258254766464233, 0.19753095507621765]}, {"step": 6300, "epoch": 13, "train_loss": 0.0013569598086178303, "cos_r_negg": [0.21028536558151245, 0.23214983940124512, 0.2632784843444824], "cos_innovation_negg": [0.21028536558151245, 0.23214983940124512, 0.2632784843444824], "cos_apical_negg": [0.006486337631940842, 0.04463715851306915, 0.08417961001396179], "cos_Ac_negg": [0.602252721786499, 0.6572246551513672, 0.6402424573898315], "r_norm": [0.19806444644927979, 0.19515500962734222, 0.2000543177127838]}, {"step": 6400, "epoch": 13, "train_loss": 0.00040100750629790127, "cos_r_negg": [0.21405652165412903, 0.23466511070728302, 0.26365357637405396], "cos_innovation_negg": [0.21405652165412903, 0.23466511070728302, 0.26365357637405396], "cos_apical_negg": [0.008446392603218555, 0.046153582632541656, 0.085267573595047], "cos_Ac_negg": [0.6010825634002686, 0.6574031114578247, 0.6403003931045532], "r_norm": [0.19844256341457367, 0.1955283284187317, 0.2004726529121399]}, {"step": 6500, "epoch": 13, "train_loss": 0.00013707700418308377, "cos_r_negg": [0.21101930737495422, 0.2340274453163147, 0.26236242055892944], "cos_innovation_negg": [0.21101930737495422, 0.2340274453163147, 0.26236242055892944], "cos_apical_negg": [0.0074137235060334206, 0.04547731950879097, 0.08196429908275604], "cos_Ac_negg": [0.5996854901313782, 0.6558645963668823, 0.6396642923355103], "r_norm": [0.19560542702674866, 0.19276125729084015, 0.19759288430213928], "ldr": {"dL_sdil": -0.012395992875099182, "dL_bp": -0.02577637881040573, "ratio": 0.480905132806902}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9745, "test_loss": 0.1421777862548828}, {"step": 6600, "epoch": 14, "train_loss": 0.0003673648461699486, "cos_r_negg": [0.20384685695171356, 0.22617018222808838, 0.25491297245025635], "cos_innovation_negg": [0.20384685695171356, 0.22617018222808838, 0.25491297245025635], "cos_apical_negg": [0.007694783620536327, 0.04521997645497322, 0.08155819773674011], "cos_Ac_negg": [0.6003413200378418, 0.6558160781860352, 0.6403923034667969], "r_norm": [0.19539327919483185, 0.19239622354507446, 0.19721531867980957]}, {"step": 6700, "epoch": 14, "train_loss": 0.0005099893314763904, "cos_r_negg": [0.2055099457502365, 0.2272024303674698, 0.25549423694610596], "cos_innovation_negg": [0.2055099457502365, 0.2272024303674698, 0.25549423694610596], "cos_apical_negg": [0.008829940110445023, 0.045250680297613144, 0.08019733428955078], "cos_Ac_negg": [0.5977998971939087, 0.6545171141624451, 0.6388810873031616], "r_norm": [0.19672875106334686, 0.1939796656370163, 0.19889411330223083]}, {"step": 6800, "epoch": 14, "train_loss": 0.0005156504339538515, "cos_r_negg": [0.19749385118484497, 0.2178269326686859, 0.24622316658496857], "cos_innovation_negg": [0.19749385118484497, 0.2178269326686859, 0.24622316658496857], "cos_apical_negg": [0.00926391500979662, 0.043578363955020905, 0.0787520557641983], "cos_Ac_negg": [0.5952610969543457, 0.6540374159812927, 0.6387388706207275], "r_norm": [0.19368970394134521, 0.19087189435958862, 0.19562256336212158]}, {"step": 6900, "epoch": 14, "train_loss": 0.0004047742404509336, "cos_r_negg": [0.19575990736484528, 0.21598784625530243, 0.2438393384218216], "cos_innovation_negg": [0.19575990736484528, 0.21598784625530243, 0.2438393384218216], "cos_apical_negg": [0.008007603697478771, 0.043969012796878815, 0.07669824361801147], "cos_Ac_negg": [0.5954526662826538, 0.6536595225334167, 0.63812255859375], "r_norm": [0.19185319542884827, 0.18893596529960632, 0.1936325877904892]}, {"step": 7000, "epoch": 14, "train_loss": 0.0006776193622499704, "cos_r_negg": [0.1946800947189331, 0.21558788418769836, 0.2416897714138031], "cos_innovation_negg": [0.1946800947189331, 0.21558788418769836, 0.2416897714138031], "cos_apical_negg": [0.005928122438490391, 0.04235561937093735, 0.07497568428516388], "cos_Ac_negg": [0.5977321863174438, 0.6545623540878296, 0.6404196619987488], "r_norm": [0.19190290570259094, 0.18885603547096252, 0.19356811046600342], "ldr": {"dL_sdil": -0.012736454606056213, "dL_bp": -0.027106821537017822, "ratio": 0.46986160249968667}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9743, "test_loss": 0.14520231761932373}], "final": {"test_acc": 0.9743, "test_loss": 0.14520231761932373, "wall_s": 22.254133224487305, "cos_r_negg": [0.1939845085144043, 0.21528545022010803, 0.24152889847755432], "cos_innovation_negg": [0.1939845085144043, 0.21528545022010803, 0.24152889847755432], "cos_apical_negg": [0.006461089476943016, 0.042778365314006805, 0.07549141347408295], "cos_Ac_negg": [0.596068799495697, 0.6536844372749329, 0.6399118900299072]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_matched_s0.json b/results/nuis_ctrl_v1_rho0p5_matched_s0.json new file mode 100644 index 0000000..f592a02 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_matched_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_matched_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.014085851609706879, -0.023326925933361053, 0.023356815800070763], "cos_innovation_negg": [-0.014816612005233765, -0.027381082996726036, 0.03303579241037369], "cos_apical_negg": [-0.014085852541029453, -0.023326924070715904, 0.023356813937425613], "cos_Ac_negg": [-0.014817332848906517, -0.027381986379623413, 0.03303509205579758], "r_norm": [4.897218704223633, 4.823971748352051, 4.820801734924316], "ldr": {"dL_sdil": 0.024374961853027344, "dL_bp": -0.19437456130981445, "ratio": -0.12540201602912424}}, {"step": 100, "epoch": 0, "train_loss": 14.12067699432373, "cos_r_negg": [-0.08893522620201111, -0.03697008267045021, -0.1882857084274292], "cos_innovation_negg": [-0.13494105637073517, -0.10561321675777435, -0.015781689435243607], "cos_apical_negg": [-0.08893522620201111, -0.03697008267045021, -0.1882856786251068], "cos_Ac_negg": [-0.13494354486465454, -0.10560338944196701, -0.01577790081501007], "r_norm": [5.334836959838867, 5.221025466918945, 5.233641624450684]}, {"step": 200, "epoch": 0, "train_loss": 6.279884338378906, "cos_r_negg": [-0.016945574432611465, -0.1910644769668579, 0.14604252576828003], "cos_innovation_negg": [-0.04803919792175293, -0.15461909770965576, -0.07992690801620483], "cos_apical_negg": [-0.016945574432611465, -0.1910644918680191, 0.14604251086711884], "cos_Ac_negg": [-0.04803498834371567, -0.1546301543712616, -0.07992991805076599], "r_norm": [5.395120620727539, 5.394388198852539, 5.436290740966797]}, {"step": 300, "epoch": 0, "train_loss": 6.813488006591797, "cos_r_negg": [0.008358773775398731, -0.06311704963445663, -0.42356497049331665], "cos_innovation_negg": [-0.00030633993446826935, 0.1247933954000473, -0.5285708904266357], "cos_apical_negg": [0.008358772844076157, -0.06311704963445663, -0.42356497049331665], "cos_Ac_negg": [-0.0003166506066918373, 0.12479470670223236, -0.5285787582397461], "r_norm": [4.722325325012207, 4.77194881439209, 4.934689521789551]}, {"step": 400, "epoch": 0, "train_loss": 16.24858856201172, "cos_r_negg": [0.015206405892968178, 0.057924140244722366, -0.7462772727012634], "cos_innovation_negg": [0.03806964308023453, 0.2180238962173462, -0.6620335578918457], "cos_apical_negg": [0.015206404030323029, 0.057924140244722366, -0.7462772727012634], "cos_Ac_negg": [0.03806009143590927, 0.21802525222301483, -0.6620345115661621], "r_norm": [4.840358734130859, 4.874863147735596, 4.971100807189941]}, {"epoch_end": 0, "step": 469, "test_acc": 0.1032, "test_loss": 9.939487109375}, {"step": 500, "epoch": 1, "train_loss": 10.660843849182129, "cos_r_negg": [-0.030392784625291824, 0.015496436506509781, -0.47973573207855225], "cos_innovation_negg": [-0.0036322539672255516, 0.02711421065032482, -0.4922116696834564], "cos_apical_negg": [-0.030392788350582123, 0.01549642812460661, -0.47973573207855225], "cos_Ac_negg": [-0.0036450987681746483, 0.027112293988466263, -0.4922292232513428], "r_norm": [4.368741035461426, 4.551837921142578, 4.542548656463623], "ldr": {"dL_sdil": -2.4881644248962402, "dL_bp": -2.4881644248962402, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 17.262676239013672, "cos_r_negg": [0.05464954674243927, 0.11473499238491058, -0.7554186582565308], "cos_innovation_negg": [0.09729184210300446, 0.26847589015960693, -0.8807673454284668], "cos_apical_negg": [0.05464954301714897, 0.11473499238491058, -0.7554186582565308], "cos_Ac_negg": [0.09740432351827621, 0.2682653069496155, -0.8807319402694702], "r_norm": [4.772709846496582, 4.876379013061523, 4.974851131439209]}, {"step": 700, "epoch": 1, "train_loss": 8.635671615600586, "cos_r_negg": [0.0331491120159626, -0.1384372115135193, -0.2274186909198761], "cos_innovation_negg": [0.03758326172828674, 0.11557825654745102, -0.2767249047756195], "cos_apical_negg": [0.0331491082906723, -0.1384372115135193, -0.22741873562335968], "cos_Ac_negg": [0.03757789731025696, 0.11556895822286606, -0.27672046422958374], "r_norm": [3.8850667476654053, 4.068549156188965, 4.125247001647949]}, {"step": 800, "epoch": 1, "train_loss": 11.105262756347656, "cos_r_negg": [-0.01403407845646143, -0.1611066460609436, -0.5888252854347229], "cos_innovation_negg": [-0.0699152946472168, 0.0787331610918045, -0.44235607981681824], "cos_apical_negg": [-0.014034079387784004, -0.1611066460609436, -0.5888252854347229], "cos_Ac_negg": [-0.06996230781078339, 0.07870141416788101, -0.44287019968032837], "r_norm": [4.181976318359375, 4.552711486816406, 4.648576259613037]}, {"step": 900, "epoch": 1, "train_loss": 19.779621124267578, "cos_r_negg": [0.029799360781908035, 0.02793087065219879, -0.7476145029067993], "cos_innovation_negg": [0.014213008806109428, 0.17063698172569275, -0.8503620028495789], "cos_apical_negg": [0.029799360781908035, 0.027930868789553642, -0.7476145029067993], "cos_Ac_negg": [0.014216138049960136, 0.17062827944755554, -0.8503645062446594], "r_norm": [3.8990087509155273, 4.121850967407227, 4.2717437744140625]}, {"epoch_end": 1, "step": 938, "test_acc": 0.0958, "test_loss": 11.862362109375}, {"step": 1000, "epoch": 2, "train_loss": 10.53783130645752, "cos_r_negg": [0.010093072429299355, -0.10424265265464783, -0.30385899543762207], "cos_innovation_negg": [0.01212595496326685, 0.10739118605852127, 0.33052748441696167], "cos_apical_negg": [0.010093078948557377, -0.10424265265464783, -0.30385899543762207], "cos_Ac_negg": [0.012088452465832233, 0.10737468302249908, 0.3305143415927887], "r_norm": [3.548647880554199, 3.817204236984253, 3.8774211406707764], "ldr": {"dL_sdil": -2.6754751205444336, "dL_bp": -2.6754751205444336, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 6.809667110443115, "cos_r_negg": [0.008733958005905151, -0.1590530127286911, -0.2897493839263916], "cos_innovation_negg": [-0.0036993511021137238, 0.0724545270204544, 0.0980292409658432], "cos_apical_negg": [0.008733958937227726, -0.1590530127286911, -0.2897493839263916], "cos_Ac_negg": [-0.0036990325897932053, 0.07245470583438873, 0.09802064299583435], "r_norm": [3.1249608993530273, 3.4257116317749023, 3.528402328491211]}, {"step": 1200, "epoch": 2, "train_loss": 7.039065361022949, "cos_r_negg": [0.00349610298871994, -0.13462018966674805, -0.32335054874420166], "cos_innovation_negg": [0.003648330457508564, 0.1927134394645691, 0.45836472511291504], "cos_apical_negg": [0.003496110439300537, -0.13462020456790924, -0.32335054874420166], "cos_Ac_negg": [0.0036400258541107178, 0.192709818482399, 0.458362340927124], "r_norm": [2.8591742515563965, 3.1747121810913086, 3.326098918914795]}, {"step": 1300, "epoch": 2, "train_loss": 16.58791160583496, "cos_r_negg": [0.019751064479351044, -0.12760673463344574, -0.17038008570671082], "cos_innovation_negg": [-0.10753948986530304, -0.033780913800001144, 0.3867144286632538], "cos_apical_negg": [0.019751064479351044, -0.12760673463344574, -0.17038008570671082], "cos_Ac_negg": [-0.10760588943958282, -0.032932430505752563, 0.386658251285553], "r_norm": [3.4449684619903564, 3.7542734146118164, 3.8927717208862305]}, {"step": 1400, "epoch": 2, "train_loss": 9.447040557861328, "cos_r_negg": [-0.03823351860046387, -0.01512721087783575, 0.011477387510240078], "cos_innovation_negg": [-0.12094312906265259, -0.11372079700231552, 0.4000576138496399], "cos_apical_negg": [-0.03823351860046387, -0.015127215534448624, 0.011477393098175526], "cos_Ac_negg": [-0.12094002962112427, -0.11377760767936707, 0.4000414311885834], "r_norm": [3.286607265472412, 3.6098904609680176, 3.777514696121216]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.1135, "test_loss": 7.29462744140625}, {"step": 1500, "epoch": 3, "train_loss": 5.781383991241455, "cos_r_negg": [0.020791932940483093, -0.13320615887641907, -0.16981716454029083], "cos_innovation_negg": [-0.03493937477469444, 0.06559880077838898, 0.12040971219539642], "cos_apical_negg": [0.02079193666577339, -0.13320615887641907, -0.16981717944145203], "cos_Ac_negg": [-0.03493931144475937, 0.06561059504747391, 0.12040558457374573], "r_norm": [2.5247819423675537, 2.8460168838500977, 3.0592761039733887], "ldr": {"dL_sdil": -0.8258872032165527, "dL_bp": -0.8258872032165527, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 16.355484008789062, "cos_r_negg": [-0.04282370209693909, -0.020879706367850304, -0.1283104419708252], "cos_innovation_negg": [-0.08435942977666855, 0.0706423819065094, 0.4163171350955963], "cos_apical_negg": [-0.04282370209693909, -0.020879698917269707, -0.128310427069664], "cos_Ac_negg": [-0.08437368273735046, 0.07062622904777527, 0.4163191616535187], "r_norm": [2.5536551475524902, 2.9975080490112305, 3.2721755504608154]}, {"step": 1700, "epoch": 3, "train_loss": 8.543461799621582, "cos_r_negg": [0.028911905363202095, -0.2613047957420349, -0.6615996360778809], "cos_innovation_negg": [0.004924428649246693, 0.2569238841533661, -0.7029926776885986], "cos_apical_negg": [0.028911907225847244, -0.2613047957420349, -0.6615996360778809], "cos_Ac_negg": [0.004948281683027744, 0.2566157579421997, -0.7029728293418884], "r_norm": [2.8632616996765137, 3.235999345779419, 3.616844654083252]}, {"step": 1800, "epoch": 3, "train_loss": 12.42558479309082, "cos_r_negg": [-0.004925549030303955, -0.05030108243227005, -0.07277588546276093], "cos_innovation_negg": [-0.06332477927207947, 0.0009691258892416954, 0.5940377712249756], "cos_apical_negg": [-0.004925550892949104, -0.05030107498168945, -0.07277587056159973], "cos_Ac_negg": [-0.06333114206790924, 0.00096840038895607, 0.594037652015686], "r_norm": [2.397101879119873, 2.8351240158081055, 3.179964780807495]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.0982, "test_loss": 9.24165390625}, {"step": 1900, "epoch": 4, "train_loss": 8.982902526855469, "cos_r_negg": [0.04248849302530289, -0.1473674476146698, -0.6803628206253052], "cos_innovation_negg": [-0.031519852578639984, 0.29750296473503113, -0.10108450055122375], "cos_apical_negg": [0.042488500475883484, -0.1473674476146698, -0.6803627610206604], "cos_Ac_negg": [-0.031492680311203, 0.29756370186805725, -0.10105365514755249], "r_norm": [2.5877840518951416, 3.036332368850708, 3.6353697776794434]}, {"step": 2000, "epoch": 4, "train_loss": 8.146219253540039, "cos_r_negg": [0.004830482415854931, 0.005079662427306175, 0.08707480132579803], "cos_innovation_negg": [-0.10026074945926666, -0.09354148805141449, -0.011778469197452068], "cos_apical_negg": [0.00483048427850008, 0.0050796642899513245, 0.08707478642463684], "cos_Ac_negg": [-0.10025837272405624, -0.09353674203157425, -0.011780465953052044], "r_norm": [2.274287223815918, 2.65627384185791, 3.059190273284912], "ldr": {"dL_sdil": -1.7607216835021973, "dL_bp": -1.7607216835021973, "ratio": 1.0}}, {"step": 2100, "epoch": 4, "train_loss": 9.06846809387207, "cos_r_negg": [-0.058182720094919205, 0.10768751055002213, 0.3173576891422272], "cos_innovation_negg": [-0.0582716166973114, -0.4040052890777588, 0.021111642941832542], "cos_apical_negg": [-0.058182716369628906, 0.10768751800060272, 0.3173576891422272], "cos_Ac_negg": [-0.058245159685611725, -0.40405023097991943, 0.021092871204018593], "r_norm": [2.4221482276916504, 2.8851242065429688, 3.391139030456543]}, {"step": 2200, "epoch": 4, "train_loss": 7.488105773925781, "cos_r_negg": [-0.009678144007921219, -0.04645390063524246, -0.060090161859989166], "cos_innovation_negg": [-0.09081825613975525, 0.19830116629600525, 0.029655873775482178], "cos_apical_negg": [-0.009678145870566368, -0.046453896909952164, -0.060090165585279465], "cos_Ac_negg": [-0.09081196784973145, 0.19828739762306213, 0.029664281755685806], "r_norm": [2.3894410133361816, 2.8490586280822754, 3.293402671813965]}, {"step": 2300, "epoch": 4, "train_loss": 8.962624549865723, "cos_r_negg": [0.01311976183205843, 0.022652335464954376, 0.09240749478340149], "cos_innovation_negg": [-0.06362192332744598, 0.005196608603000641, 0.35803020000457764], "cos_apical_negg": [0.013119753450155258, 0.022652335464954376, 0.09240749478340149], "cos_Ac_negg": [-0.06361232697963715, 0.0052616894245147705, 0.3580298125743866], "r_norm": [1.936016321182251, 2.3289709091186523, 2.7761802673339844]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.1032, "test_loss": 6.76871787109375}, {"step": 2400, "epoch": 5, "train_loss": 4.417475700378418, "cos_r_negg": [-0.030662287026643753, 0.05300325155258179, -0.020648101344704628], "cos_innovation_negg": [-0.08473651856184006, 0.024036895483732224, 0.25856074690818787], "cos_apical_negg": [-0.030662287026643753, 0.05300324410200119, -0.02064809761941433], "cos_Ac_negg": [-0.08473721891641617, 0.02404254674911499, 0.25856029987335205], "r_norm": [1.8500384092330933, 2.164245128631592, 2.623790979385376]}, {"step": 2500, "epoch": 5, "train_loss": 6.71903133392334, "cos_r_negg": [-0.02484159916639328, 0.12425047159194946, 0.0795184075832367], "cos_innovation_negg": [-0.04391147941350937, -0.1385723054409027, 0.24954695999622345], "cos_apical_negg": [-0.02484159916639328, 0.12425047159194946, 0.07951842248439789], "cos_Ac_negg": [-0.04390912503004074, -0.1385713815689087, 0.24954679608345032], "r_norm": [1.8559162616729736, 2.264186382293701, 2.7719039916992188], "ldr": {"dL_sdil": -1.6567621231079102, "dL_bp": -1.6567621231079102, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 15.13719654083252, "cos_r_negg": [-0.030691301450133324, -0.09046292304992676, -0.4273899793624878], "cos_innovation_negg": [-0.07379123568534851, -0.09459459036588669, 0.20217010378837585], "cos_apical_negg": [-0.030691301450133324, -0.09046293050050735, -0.4273899793624878], "cos_Ac_negg": [-0.07372713088989258, -0.10801927000284195, 0.20215289294719696], "r_norm": [2.205949068069458, 2.7041139602661133, 3.242435932159424]}, {"step": 2700, "epoch": 5, "train_loss": 13.799510955810547, "cos_r_negg": [-0.00035359058529138565, 0.01679469831287861, -0.18613708019256592], "cos_innovation_negg": [-0.08058209717273712, 0.191128671169281, 0.17677849531173706], "cos_apical_negg": [-0.00035359105095267296, 0.016794709488749504, -0.1861370950937271], "cos_Ac_negg": [-0.08057533949613571, 0.19111774861812592, 0.1767803430557251], "r_norm": [2.214383602142334, 2.621917963027954, 3.161092758178711]}, {"step": 2800, "epoch": 5, "train_loss": 9.245009422302246, "cos_r_negg": [-0.012889944016933441, 0.10436075925827026, -0.6796268224716187], "cos_innovation_negg": [-0.06484337151050568, 0.4044119715690613, -0.0012653958983719349], "cos_apical_negg": [-0.01288994774222374, 0.10436075925827026, -0.6796268224716187], "cos_Ac_negg": [-0.06484868377447128, 0.4044126272201538, -0.0012657511979341507], "r_norm": [1.9268364906311035, 2.292642116546631, 3.1016478538513184]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.1009, "test_loss": 9.69221904296875}, {"step": 2900, "epoch": 6, "train_loss": 13.638516426086426, "cos_r_negg": [0.02191525138914585, -0.040097691118717194, -0.642638087272644], "cos_innovation_negg": [-0.04706123471260071, 0.3079439103603363, -0.48477670550346375], "cos_apical_negg": [0.021915253251791, -0.0400976836681366, -0.642638087272644], "cos_Ac_negg": [-0.0470699742436409, 0.3078739047050476, -0.4847766160964966], "r_norm": [1.9580109119415283, 2.3436472415924072, 3.148317813873291]}, {"step": 3000, "epoch": 6, "train_loss": 10.292101860046387, "cos_r_negg": [0.053298208862543106, -0.07172314822673798, 0.057993773370981216], "cos_innovation_negg": [-0.002374793402850628, -0.07622523605823517, 0.8782966136932373], "cos_apical_negg": [0.05329820141196251, -0.07172315567731857, 0.05799376964569092], "cos_Ac_negg": [-0.0022748056799173355, -0.07616179436445236, 0.8782958984375], "r_norm": [1.9307631254196167, 2.231853723526001, 3.164006233215332], "ldr": {"dL_sdil": -4.221527099609375, "dL_bp": -4.221527099609375, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 13.738449096679688, "cos_r_negg": [0.04281838238239288, -0.025666138157248497, 0.06642301380634308], "cos_innovation_negg": [-0.006224539130926132, -0.08096176385879517, 0.8056594133377075], "cos_apical_negg": [0.04281838983297348, -0.025666140019893646, 0.06642301380634308], "cos_Ac_negg": [-0.005950918421149254, -0.08089739084243774, 0.8056603670120239], "r_norm": [1.89373779296875, 2.186147689819336, 3.1485190391540527]}, {"step": 3200, "epoch": 6, "train_loss": 6.55476188659668, "cos_r_negg": [-0.03421996161341667, -0.2742629647254944, -0.09308277070522308], "cos_innovation_negg": [0.04799528047442436, 0.38088029623031616, 0.461800217628479], "cos_apical_negg": [-0.03421995788812637, -0.274262934923172, -0.09308278560638428], "cos_Ac_negg": [0.048000581562519073, 0.38088715076446533, 0.4618002772331238], "r_norm": [1.605065941810608, 1.9066911935806274, 2.701836109161377]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.1032, "test_loss": 15.26507431640625}, {"step": 3300, "epoch": 7, "train_loss": 10.61668586730957, "cos_r_negg": [-0.03434322774410248, 0.032522112131118774, 0.005781191401183605], "cos_innovation_negg": [-0.07201001793146133, -0.16277408599853516, 0.2681025266647339], "cos_apical_negg": [-0.03434322774410248, 0.032522112131118774, 0.005781190469861031], "cos_Ac_negg": [-0.072011798620224, -0.16275878250598907, 0.2681030035018921], "r_norm": [1.7817790508270264, 2.1636571884155273, 2.914649724960327]}, {"step": 3400, "epoch": 7, "train_loss": 9.504243850708008, "cos_r_negg": [0.019950853660702705, -0.20116356015205383, 0.2762332260608673], "cos_innovation_negg": [0.08255147188901901, 0.31720560789108276, 0.6831055879592896], "cos_apical_negg": [0.019950851798057556, -0.20116353034973145, 0.2762332260608673], "cos_Ac_negg": [0.08255710452795029, 0.3172174394130707, 0.6831055879592896], "r_norm": [1.677664041519165, 1.9817421436309814, 2.7234790325164795]}, {"step": 3500, "epoch": 7, "train_loss": 13.170604705810547, "cos_r_negg": [-0.028093859553337097, -0.2691152095794678, -0.17662903666496277], "cos_innovation_negg": [-0.009174267761409283, 0.24669218063354492, 0.634959876537323], "cos_apical_negg": [-0.028093859553337097, -0.26911523938179016, -0.17662903666496277], "cos_Ac_negg": [-0.009167949669063091, 0.24664577841758728, 0.6349596381187439], "r_norm": [1.6037486791610718, 1.8770670890808105, 2.861909866333008], "ldr": {"dL_sdil": -1.423675537109375, "dL_bp": -1.423675537109375, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 11.314979553222656, "cos_r_negg": [-0.011983984149992466, -0.07089132815599442, -0.5297096967697144], "cos_innovation_negg": [-0.16919922828674316, 0.16218727827072144, 0.07026683539152145], "cos_apical_negg": [-0.011983980424702168, -0.07089132070541382, -0.5297096967697144], "cos_Ac_negg": [-0.1692042201757431, 0.16216658055782318, 0.0702667385339737], "r_norm": [1.60227370262146, 1.8934738636016846, 2.7727532386779785]}, {"step": 3700, "epoch": 7, "train_loss": 7.798266410827637, "cos_r_negg": [-0.01820511370897293, -0.03391684591770172, -8.560018613934517e-05], "cos_innovation_negg": [-0.08299519121646881, 0.05394937843084335, 0.4331936836242676], "cos_apical_negg": [-0.01820511184632778, -0.033916838467121124, -8.558016270399094e-05], "cos_Ac_negg": [-0.08299021422863007, 0.053952451795339584, 0.4331936836242676], "r_norm": [1.6263720989227295, 1.9731025695800781, 2.831577777862549]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.1028, "test_loss": 7.502961474609375}, {"step": 3800, "epoch": 8, "train_loss": 9.374402046203613, "cos_r_negg": [-0.007190410979092121, -0.14519503712654114, 0.3474685549736023], "cos_innovation_negg": [-0.04825960099697113, 0.348207026720047, 0.47952884435653687], "cos_apical_negg": [-0.007190404459834099, -0.14519503712654114, 0.3474685847759247], "cos_Ac_negg": [-0.048091258853673935, 0.34822291135787964, 0.47952908277511597], "r_norm": [1.6815733909606934, 2.0130138397216797, 3.1404061317443848]}, {"step": 3900, "epoch": 8, "train_loss": 7.552352428436279, "cos_r_negg": [-0.013247175142168999, -0.07759027183055878, -0.19268079102039337], "cos_innovation_negg": [-0.012854320928454399, 0.013200514018535614, 0.6624311208724976], "cos_apical_negg": [-0.01324717327952385, -0.07759027183055878, -0.19268079102039337], "cos_Ac_negg": [-0.012856891378760338, 0.01319839246571064, 0.6624311208724976], "r_norm": [1.5521570444107056, 1.7970399856567383, 2.743655204772949]}, {"step": 4000, "epoch": 8, "train_loss": 7.4703474044799805, "cos_r_negg": [-0.03362344950437546, 0.03418354690074921, 0.09146559238433838], "cos_innovation_negg": [-0.05962652713060379, -0.07708211988210678, 0.5295485258102417], "cos_apical_negg": [-0.03362344950437546, 0.034183550626039505, 0.09146557748317719], "cos_Ac_negg": [-0.05962176248431206, -0.07708053290843964, 0.5295485258102417], "r_norm": [1.3777050971984863, 1.6053621768951416, 2.60689640045166], "ldr": {"dL_sdil": -1.0782999992370605, "dL_bp": -1.0782999992370605, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 15.02281379699707, "cos_r_negg": [-0.002551573794335127, -0.1829811930656433, 0.016996806487441063], "cos_innovation_negg": [-0.030981209129095078, -0.03694143146276474, 0.3626062273979187], "cos_apical_negg": [-0.002551571000367403, -0.1829811930656433, 0.016996804624795914], "cos_Ac_negg": [-0.03099149465560913, -0.03701050952076912, 0.362606406211853], "r_norm": [1.6183452606201172, 1.917630672454834, 3.1797680854797363]}, {"step": 4200, "epoch": 8, "train_loss": 10.705953598022461, "cos_r_negg": [0.04023492708802223, -0.28198927640914917, -0.5521667003631592], "cos_innovation_negg": [-0.061573825776576996, 0.23431363701820374, 0.49464771151542664], "cos_apical_negg": [0.040234923362731934, -0.28198930621147156, -0.5521667003631592], "cos_Ac_negg": [-0.06158052012324333, 0.23431259393692017, 0.49464765191078186], "r_norm": [1.4082860946655273, 1.6171205043792725, 2.7632203102111816]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.1009, "test_loss": 10.2951087890625}, {"step": 4300, "epoch": 9, "train_loss": 19.057186126708984, "cos_r_negg": [-0.038887206465005875, -0.10910739749670029, -0.125909224152565], "cos_innovation_negg": [-0.14409834146499634, 0.004107946529984474, 0.5602022409439087], "cos_apical_negg": [-0.03888721764087677, -0.10910740494728088, -0.125909224152565], "cos_Ac_negg": [-0.1441044807434082, 0.00409357063472271, 0.5602022409439087], "r_norm": [1.558234453201294, 1.8885302543640137, 2.955050468444824]}, {"step": 4400, "epoch": 9, "train_loss": 9.278549194335938, "cos_r_negg": [-0.0016661696135997772, -0.162213534116745, -0.3255435824394226], "cos_innovation_negg": [-0.07545366883277893, -0.06068694218993187, 0.3472710847854614], "cos_apical_negg": [-0.0016661710105836391, -0.162213534116745, -0.3255435824394226], "cos_Ac_negg": [-0.0754532516002655, -0.06069668382406235, 0.3472710847854614], "r_norm": [1.4222195148468018, 1.6736817359924316, 2.760430335998535]}, {"step": 4500, "epoch": 9, "train_loss": 10.898351669311523, "cos_r_negg": [0.004468430764973164, -0.3113348186016083, -0.2942622900009155], "cos_innovation_negg": [-0.0989055186510086, 0.07418051362037659, 0.5252222418785095], "cos_apical_negg": [0.004468428902328014, -0.3113348186016083, -0.2942622900009155], "cos_Ac_negg": [-0.09891639649868011, 0.07416218519210815, 0.5252221822738647], "r_norm": [1.3769478797912598, 1.6362826824188232, 2.7787399291992188], "ldr": {"dL_sdil": -3.5822229385375977, "dL_bp": -3.5822229385375977, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 7.632448196411133, "cos_r_negg": [-0.01497666072100401, -0.12002019584178925, -0.19787827134132385], "cos_innovation_negg": [-0.11181139945983887, 0.01905500888824463, 0.7482896447181702], "cos_apical_negg": [-0.014976659789681435, -0.12002018839120865, -0.19787827134132385], "cos_Ac_negg": [-0.11183474212884903, 0.019033975899219513, 0.7482897043228149], "r_norm": [1.5103447437286377, 1.7593655586242676, 3.0423498153686523]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.101, "test_loss": 17.489161328125}, {"step": 4700, "epoch": 10, "train_loss": 10.94247055053711, "cos_r_negg": [0.01760529913008213, 0.04373510181903839, -0.22820879518985748], "cos_innovation_negg": [-0.03442804515361786, -0.04907580837607384, 0.5208080410957336], "cos_apical_negg": [0.01760529726743698, 0.04373509809374809, -0.22820882499217987], "cos_Ac_negg": [-0.03444037586450577, -0.04908207058906555, 0.5208081007003784], "r_norm": [1.4914650917053223, 1.8365864753723145, 2.999756097793579]}, {"step": 4800, "epoch": 10, "train_loss": 8.38232707977295, "cos_r_negg": [-0.022430866956710815, 0.06743571907281876, 0.14289698004722595], "cos_innovation_negg": [-0.061999671161174774, -0.08354751765727997, 0.5676090121269226], "cos_apical_negg": [-0.022430866956710815, 0.06743571162223816, 0.14289695024490356], "cos_Ac_negg": [-0.062002383172512054, -0.08352405577898026, 0.5676090121269226], "r_norm": [1.2851887941360474, 1.4712836742401123, 2.768702507019043]}, {"step": 4900, "epoch": 10, "train_loss": 14.846281051635742, "cos_r_negg": [-0.016718782484531403, -0.060600996017456055, -0.3261968791484833], "cos_innovation_negg": [-0.08924563229084015, 0.018774142488837242, 0.3928990960121155], "cos_apical_negg": [-0.016718782484531403, -0.06060100346803665, -0.3261968493461609], "cos_Ac_negg": [-0.08925706893205643, 0.018773484975099564, 0.39289915561676025], "r_norm": [1.3493835926055908, 1.5683330297470093, 3.0793542861938477]}, {"step": 5000, "epoch": 10, "train_loss": 14.356071472167969, "cos_r_negg": [0.008912526071071625, -0.49663084745407104, -0.04299021512269974], "cos_innovation_negg": [-0.06435373425483704, 0.13328802585601807, 0.741351842880249], "cos_apical_negg": [0.008912526071071625, -0.49663084745407104, -0.04299021512269974], "cos_Ac_negg": [-0.0643562600016594, 0.13328857719898224, 0.741351842880249], "r_norm": [1.3121259212493896, 1.4445512294769287, 2.8401119709014893], "ldr": {"dL_sdil": -2.009305953979492, "dL_bp": -2.009305953979492, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 14.768389701843262, "cos_r_negg": [0.011822660453617573, -0.21893084049224854, -0.2701816260814667], "cos_innovation_negg": [-0.11576712131500244, -0.039966557174921036, 0.42145782709121704], "cos_apical_negg": [0.011822663247585297, -0.21893084049224854, -0.27018165588378906], "cos_Ac_negg": [-0.11577373743057251, -0.03996579349040985, 0.42145782709121704], "r_norm": [1.3573578596115112, 1.5708760023117065, 2.779130697250366]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.1009, "test_loss": 7.66514248046875}, {"step": 5200, "epoch": 11, "train_loss": 9.388562202453613, "cos_r_negg": [0.02991987206041813, -0.15401536226272583, -0.18005609512329102], "cos_innovation_negg": [-0.12797051668167114, -0.11414825171232224, 0.7665873169898987], "cos_apical_negg": [0.029919877648353577, -0.15401536226272583, -0.18005609512329102], "cos_Ac_negg": [-0.12800925970077515, -0.11416096240282059, 0.7665873169898987], "r_norm": [1.3155336380004883, 1.4995043277740479, 2.9162590503692627]}, {"step": 5300, "epoch": 11, "train_loss": 16.368091583251953, "cos_r_negg": [0.01324643287807703, -0.2048347443342209, -0.11145533621311188], "cos_innovation_negg": [-0.03745388239622116, -0.1980554163455963, 0.879595160484314], "cos_apical_negg": [0.013246431015431881, -0.2048347294330597, -0.11145532131195068], "cos_Ac_negg": [-0.03742729872465134, -0.19806262850761414, 0.8795951008796692], "r_norm": [1.3567450046539307, 1.4792323112487793, 3.1258230209350586]}, {"step": 5400, "epoch": 11, "train_loss": 9.241300582885742, "cos_r_negg": [-0.002415075898170471, -0.062397465109825134, -0.1283145248889923], "cos_innovation_negg": [-0.12424810230731964, -0.11525802314281464, 0.8398817777633667], "cos_apical_negg": [-0.0024150805547833443, -0.062397465109825134, -0.1283145397901535], "cos_Ac_negg": [-0.12426209449768066, -0.11525819450616837, 0.8398817777633667], "r_norm": [1.1977722644805908, 1.3680554628372192, 2.795916795730591]}, {"step": 5500, "epoch": 11, "train_loss": 7.1277337074279785, "cos_r_negg": [0.020042669028043747, -0.19323642551898956, -0.09448372572660446], "cos_innovation_negg": [-0.054096072912216187, 0.07144784927368164, 0.7631316184997559], "cos_apical_negg": [0.020042672753334045, -0.19323642551898956, -0.09448373317718506], "cos_Ac_negg": [-0.05410517007112503, 0.07144825905561447, 0.7631316184997559], "r_norm": [1.2511935234069824, 1.508116602897644, 2.847163677215576], "ldr": {"dL_sdil": -1.179067611694336, "dL_bp": -1.179067611694336, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 9.222949981689453, "cos_r_negg": [0.04146236553788185, -0.12034913897514343, -0.201630637049675], "cos_innovation_negg": [0.00920295063406229, 0.19425688683986664, 0.8021167516708374], "cos_apical_negg": [0.04146236926317215, -0.12034913897514343, -0.201630637049675], "cos_Ac_negg": [0.009195610880851746, 0.19425855576992035, 0.8021166920661926], "r_norm": [1.2221951484680176, 1.3334851264953613, 2.826223373413086]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.1135, "test_loss": 8.62473544921875}, {"step": 5700, "epoch": 12, "train_loss": 15.389667510986328, "cos_r_negg": [0.009794110432267189, -0.2861332297325134, -0.40324753522872925], "cos_innovation_negg": [-0.04145323485136032, 0.015989365056157112, 0.3912540078163147], "cos_apical_negg": [0.009794116020202637, -0.2861332297325134, -0.40324753522872925], "cos_Ac_negg": [-0.041419871151447296, 0.015989385545253754, 0.3912540078163147], "r_norm": [1.4236292839050293, 1.7887077331542969, 3.2164554595947266]}, {"step": 5800, "epoch": 12, "train_loss": 11.608987808227539, "cos_r_negg": [0.0264580175280571, -0.18739908933639526, 0.22039225697517395], "cos_innovation_negg": [0.06749127805233002, 0.1761072427034378, 0.540614664554596], "cos_apical_negg": [0.0264580100774765, -0.18739910423755646, 0.22039225697517395], "cos_Ac_negg": [0.06724525988101959, 0.17611561715602875, 0.540614902973175], "r_norm": [1.3640036582946777, 1.523836374282837, 3.325641632080078]}, {"step": 5900, "epoch": 12, "train_loss": 9.461895942687988, "cos_r_negg": [0.017362043261528015, -0.1316436231136322, 0.059619009494781494], "cos_innovation_negg": [-0.06479247659444809, -0.13603007793426514, 0.8205447196960449], "cos_apical_negg": [0.017362050712108612, -0.1316436231136322, 0.05961896479129791], "cos_Ac_negg": [-0.06478221714496613, -0.13603004813194275, 0.8205447196960449], "r_norm": [1.1654126644134521, 1.3804471492767334, 2.7644495964050293]}, {"step": 6000, "epoch": 12, "train_loss": 7.1293134689331055, "cos_r_negg": [0.00018607545644044876, 0.026982855051755905, 0.19737862050533295], "cos_innovation_negg": [0.1266070008277893, 0.18609347939491272, 0.6317232847213745], "cos_apical_negg": [0.00018606707453727722, 0.026982856914401054, 0.19737860560417175], "cos_Ac_negg": [0.12655681371688843, 0.1860934942960739, 0.6317233443260193], "r_norm": [1.4334667921066284, 1.6781846284866333, 3.1051998138427734], "ldr": {"dL_sdil": -1.5357465744018555, "dL_bp": -1.5357465744018555, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.0982, "test_loss": 19.83318984375}, {"step": 6100, "epoch": 13, "train_loss": 17.766162872314453, "cos_r_negg": [0.033010635524988174, -0.2682313024997711, 0.20800001919269562], "cos_innovation_negg": [0.011389613151550293, 0.08941420912742615, 0.7991719245910645], "cos_apical_negg": [0.033010631799697876, -0.2682313024997711, 0.20799998939037323], "cos_Ac_negg": [0.011388595215976238, 0.08941429108381271, 0.7991719841957092], "r_norm": [1.232367753982544, 1.387908935546875, 3.0289838314056396]}, {"step": 6200, "epoch": 13, "train_loss": 11.01158332824707, "cos_r_negg": [0.01041453704237938, 0.15852689743041992, 0.3439769744873047], "cos_innovation_negg": [0.026772689074277878, -0.022208023816347122, 0.8595736026763916], "cos_apical_negg": [0.010414529591798782, 0.15852686762809753, 0.3439769744873047], "cos_Ac_negg": [0.026771195232868195, -0.02220819517970085, 0.8595736026763916], "r_norm": [1.2073860168457031, 1.3000640869140625, 3.0208113193511963]}, {"step": 6300, "epoch": 13, "train_loss": 12.948205947875977, "cos_r_negg": [-0.004232844337821007, -0.161408931016922, -0.024524416774511337], "cos_innovation_negg": [-0.05507580190896988, 0.11693840473890305, 0.6276800632476807], "cos_apical_negg": [-0.004232846200466156, -0.161408931016922, -0.024524405598640442], "cos_Ac_negg": [-0.05515195429325104, 0.11693845689296722, 0.6276801228523254], "r_norm": [1.3564059734344482, 1.736625075340271, 3.225682258605957]}, {"step": 6400, "epoch": 13, "train_loss": 10.635601043701172, "cos_r_negg": [0.04292807728052139, -0.532495379447937, -0.028853420168161392], "cos_innovation_negg": [0.0049087535589933395, 0.35492196679115295, 0.8573130965232849], "cos_apical_negg": [0.042928069829940796, -0.532495379447937, -0.0288533978164196], "cos_Ac_negg": [0.005078186746686697, 0.35492223501205444, 0.8573129773139954], "r_norm": [1.3293821811676025, 1.3493680953979492, 3.3992202281951904]}, {"step": 6500, "epoch": 13, "train_loss": 11.81856918334961, "cos_r_negg": [0.03599420189857483, -0.4529581069946289, -0.11311055719852448], "cos_innovation_negg": [-0.04727689549326897, -0.15589715540409088, 0.29647719860076904], "cos_apical_negg": [0.03599419817328453, -0.4529581069946289, -0.11311057955026627], "cos_Ac_negg": [-0.0473436638712883, -0.15589724481105804, 0.29647713899612427], "r_norm": [1.5132429599761963, 1.8024349212646484, 3.321284532546997], "ldr": {"dL_sdil": -3.3649139404296875, "dL_bp": -3.3649139404296875, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.098, "test_loss": 10.1111287109375}, {"step": 6600, "epoch": 14, "train_loss": 9.646769523620605, "cos_r_negg": [0.08268283307552338, -0.48627281188964844, -0.12591207027435303], "cos_innovation_negg": [0.05956495553255081, 0.09663169085979462, 0.8726704120635986], "cos_apical_negg": [0.08268284797668457, -0.48627281188964844, -0.12591208517551422], "cos_Ac_negg": [0.059537194669246674, 0.09663171321153641, 0.8726704120635986], "r_norm": [1.207603931427002, 1.342028260231018, 3.224560499191284]}, {"step": 6700, "epoch": 14, "train_loss": 12.739542961120605, "cos_r_negg": [0.045490365475416183, -0.37703779339790344, -0.288271427154541], "cos_innovation_negg": [-0.020781833678483963, -0.0022076843306422234, 0.5856966972351074], "cos_apical_negg": [0.045490361750125885, -0.37703779339790344, -0.288271427154541], "cos_Ac_negg": [-0.020783554762601852, -0.0022076803725212812, 0.5856967568397522], "r_norm": [1.1937823295593262, 1.3459299802780151, 2.9678592681884766]}, {"step": 6800, "epoch": 14, "train_loss": 5.33766508102417, "cos_r_negg": [0.04322122782468796, -0.2240423560142517, 0.06438862532377243], "cos_innovation_negg": [0.078287273645401, -0.05883041024208069, 0.9347652196884155], "cos_apical_negg": [0.04322122782468796, -0.2240423560142517, 0.06438861787319183], "cos_Ac_negg": [0.07829080522060394, -0.05883039906620979, 0.9347652196884155], "r_norm": [1.1590958833694458, 1.2810781002044678, 3.145021438598633]}, {"step": 6900, "epoch": 14, "train_loss": 8.997514724731445, "cos_r_negg": [-0.019797516986727715, -0.07225950807332993, 0.1420089304447174], "cos_innovation_negg": [-0.016311611980199814, -0.09956280887126923, 0.5852810740470886], "cos_apical_negg": [-0.019797515124082565, -0.07225951552391052, 0.1420089304447174], "cos_Ac_negg": [-0.01638435199856758, -0.09956278651952744, 0.5852811336517334], "r_norm": [1.4335718154907227, 1.721304178237915, 3.230219602584839]}, {"step": 7000, "epoch": 14, "train_loss": 10.408319473266602, "cos_r_negg": [-0.026550497859716415, -0.48747149109840393, -0.03160889446735382], "cos_innovation_negg": [-0.00826600193977356, 0.1541316956281662, 0.5695642828941345], "cos_apical_negg": [-0.02655049040913582, -0.48747149109840393, -0.03160889446735382], "cos_Ac_negg": [-0.008299950510263443, 0.1541317254304886, 0.5695642828941345], "r_norm": [1.102695345878601, 1.2425671815872192, 3.0119237899780273], "ldr": {"dL_sdil": -3.2128562927246094, "dL_bp": -3.2128562927246094, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1032, "test_loss": 15.020373046875}], "final": {"test_acc": 0.1032, "test_loss": 15.020373046875, "wall_s": 27.269801139831543, "cos_r_negg": [0.06974278390407562, -0.19665034115314484, 0.001963862217962742], "cos_innovation_negg": [0.06065504997968674, 0.24382783472537994, 0.677558422088623], "cos_apical_negg": [0.06974278390407562, -0.19665032625198364, 0.001963871531188488], "cos_Ac_negg": [0.06064701825380325, 0.24382787942886353, 0.6775585412979126]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_matched_s1.json b/results/nuis_ctrl_v1_rho0p5_matched_s1.json new file mode 100644 index 0000000..f735151 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_matched_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_matched_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.01368984766304493, -0.012001646682620049, -0.008926067501306534], "cos_innovation_negg": [0.026290064677596092, -0.012671238742768764, -0.006505622528493404], "cos_apical_negg": [0.01368984766304493, -0.012001646682620049, -0.008926068432629108], "cos_Ac_negg": [0.02628779411315918, -0.01267300546169281, -0.006506332661956549], "r_norm": [4.743828773498535, 4.803375244140625, 4.794331073760986], "ldr": {"dL_sdil": 0.021651029586791992, "dL_bp": -0.177276611328125, "ratio": -0.12213133714925116}}, {"step": 100, "epoch": 0, "train_loss": 8.629837036132812, "cos_r_negg": [0.037567175924777985, 0.008364958688616753, -0.5287129878997803], "cos_innovation_negg": [0.04334171861410141, 0.22261875867843628, 0.14754386246204376], "cos_apical_negg": [0.037567175924777985, 0.008364951238036156, -0.5287129282951355], "cos_Ac_negg": [0.04345659911632538, 0.22264283895492554, 0.14739736914634705], "r_norm": [5.985307693481445, 6.0046281814575195, 5.903804779052734]}, {"step": 200, "epoch": 0, "train_loss": 8.29200553894043, "cos_r_negg": [-0.05966085195541382, -0.09444378316402435, -0.09045639634132385], "cos_innovation_negg": [-0.09845882654190063, -0.06108493357896805, 0.19033609330654144], "cos_apical_negg": [-0.05966084823012352, -0.09444377571344376, -0.09045639634132385], "cos_Ac_negg": [-0.09845120459794998, -0.06108956038951874, 0.190270334482193], "r_norm": [4.763657569885254, 4.891393661499023, 4.839957237243652]}, {"step": 300, "epoch": 0, "train_loss": 14.548294067382812, "cos_r_negg": [-0.12011945992708206, -0.17529357969760895, -0.24218785762786865], "cos_innovation_negg": [-0.13343249261379242, -0.026975946500897408, 0.3038310706615448], "cos_apical_negg": [-0.12011946737766266, -0.17529356479644775, -0.24218782782554626], "cos_Ac_negg": [-0.13343918323516846, -0.02698030322790146, 0.3037066161632538], "r_norm": [4.711991786956787, 4.9175238609313965, 4.900652885437012]}, {"step": 400, "epoch": 0, "train_loss": 10.571653366088867, "cos_r_negg": [-0.07457352429628372, -0.24640749394893646, -0.37246787548065186], "cos_innovation_negg": [-0.11332274973392487, -0.2235855758190155, -0.0984620451927185], "cos_apical_negg": [-0.07457352429628372, -0.24640749394893646, -0.37246787548065186], "cos_Ac_negg": [-0.07068943977355957, -0.210202157497406, 0.04788782447576523], "r_norm": [4.917359828948975, 5.2306671142578125, 5.053191184997559]}, {"epoch_end": 0, "step": 469, "test_acc": 0.0892, "test_loss": 11.37957666015625}, {"step": 500, "epoch": 1, "train_loss": 11.507259368896484, "cos_r_negg": [0.0044118864461779594, -0.029640208929777145, -0.5220323801040649], "cos_innovation_negg": [-0.008758346550166607, 0.03197118267416954, 0.22668370604515076], "cos_apical_negg": [0.004411884117871523, -0.02964020147919655, -0.5220324397087097], "cos_Ac_negg": [-0.008773845620453358, 0.03197310492396355, 0.22666186094284058], "r_norm": [4.150497913360596, 4.425154209136963, 4.40374231338501], "ldr": {"dL_sdil": -0.8359193801879883, "dL_bp": -0.8359193801879883, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 8.379337310791016, "cos_r_negg": [-0.04307126998901367, -0.21659450232982635, -0.1836141049861908], "cos_innovation_negg": [-0.08859698474407196, 0.11656951904296875, 0.28489020466804504], "cos_apical_negg": [-0.04307126998901367, -0.21659448742866516, -0.1836141049861908], "cos_Ac_negg": [-0.08861087262630463, 0.1165703535079956, 0.28487831354141235], "r_norm": [3.9001951217651367, 4.099247932434082, 4.126674175262451]}, {"step": 700, "epoch": 1, "train_loss": 10.07689380645752, "cos_r_negg": [0.014078537002205849, 0.016958516091108322, -0.04757288843393326], "cos_innovation_negg": [2.461625263094902e-05, 0.192184180021286, 0.4833545982837677], "cos_apical_negg": [0.014078537002205849, 0.016958512365818024, -0.047572895884513855], "cos_Ac_negg": [1.161079853773117e-05, 0.192181795835495, 0.4833124279975891], "r_norm": [3.9420077800750732, 4.221686363220215, 4.267495155334473]}, {"step": 800, "epoch": 1, "train_loss": 6.442304611206055, "cos_r_negg": [0.037889618426561356, -0.06307996809482574, -0.063703253865242], "cos_innovation_negg": [-0.04553075507283211, 0.10371538996696472, 0.2536274194717407], "cos_apical_negg": [0.037889622151851654, -0.06307997554540634, -0.06370328366756439], "cos_Ac_negg": [-0.045541077852249146, 0.1037130281329155, 0.2535904049873352], "r_norm": [3.5866873264312744, 3.9869673252105713, 4.057910442352295]}, {"step": 900, "epoch": 1, "train_loss": 15.13424301147461, "cos_r_negg": [0.030607381835579872, -0.2586507201194763, -0.4286853075027466], "cos_innovation_negg": [0.014614340849220753, -0.10848779231309891, 0.6467256546020508], "cos_apical_negg": [0.03060738369822502, -0.2586507201194763, -0.4286853075027466], "cos_Ac_negg": [0.014606429263949394, -0.1084863692522049, 0.6467254757881165], "r_norm": [3.4449856281280518, 3.8409712314605713, 3.8702330589294434]}, {"epoch_end": 1, "step": 938, "test_acc": 0.0892, "test_loss": 5.97907939453125}, {"step": 1000, "epoch": 2, "train_loss": 16.947219848632812, "cos_r_negg": [-0.05575894936919212, -0.058172352612018585, -0.10303526371717453], "cos_innovation_negg": [-0.10887196660041809, -0.09698377549648285, 0.41782981157302856], "cos_apical_negg": [-0.05575894936919212, -0.058172356337308884, -0.10303527116775513], "cos_Ac_negg": [-0.1088702529668808, -0.09699638187885284, 0.4178636074066162], "r_norm": [3.4725301265716553, 3.893908977508545, 3.9083449840545654], "ldr": {"dL_sdil": -1.2378950119018555, "dL_bp": -1.2378950119018555, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 14.489168167114258, "cos_r_negg": [0.007330155000090599, -0.2059539258480072, -0.6385909914970398], "cos_innovation_negg": [-0.058647677302360535, 0.10369955003261566, 0.1521846204996109], "cos_apical_negg": [0.007330155000090599, -0.2059539258480072, -0.6385909914970398], "cos_Ac_negg": [-0.058645546436309814, 0.10369760543107986, 0.15215173363685608], "r_norm": [3.404529094696045, 3.8415870666503906, 3.890641212463379]}, {"step": 1200, "epoch": 2, "train_loss": 11.066431045532227, "cos_r_negg": [0.040565118193626404, -0.26042842864990234, -0.30299752950668335], "cos_innovation_negg": [0.061117686331272125, 0.14500866830348969, 0.5352002382278442], "cos_apical_negg": [0.040565118193626404, -0.26042845845222473, -0.30299752950668335], "cos_Ac_negg": [0.06110852211713791, 0.14500820636749268, 0.5352024435997009], "r_norm": [2.9809021949768066, 3.4198238849639893, 3.4656436443328857]}, {"step": 1300, "epoch": 2, "train_loss": 10.231103897094727, "cos_r_negg": [0.01986650936305523, -0.09248112887144089, -0.06143999099731445], "cos_innovation_negg": [0.03777393698692322, 0.3218173682689667, 0.3766866624355316], "cos_apical_negg": [0.01986650750041008, -0.09248113632202148, -0.061439987272024155], "cos_Ac_negg": [0.03776753321290016, 0.32181769609451294, 0.3766733407974243], "r_norm": [2.943350076675415, 3.4399678707122803, 3.53225040435791]}, {"step": 1400, "epoch": 2, "train_loss": 15.886367797851562, "cos_r_negg": [0.03741500899195671, -0.041028186678886414, -0.22061492502689362], "cos_innovation_negg": [0.04061534255743027, 0.5060548782348633, 0.49722927808761597], "cos_apical_negg": [0.03741501271724701, -0.04102819040417671, -0.220614954829216], "cos_Ac_negg": [0.040607623755931854, 0.5060560703277588, 0.4972347617149353], "r_norm": [2.889829635620117, 3.452561855316162, 3.5183520317077637]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.098, "test_loss": 8.59035224609375}, {"step": 1500, "epoch": 3, "train_loss": 15.093576431274414, "cos_r_negg": [0.02216712012887001, -0.21593934297561646, -0.1491767168045044], "cos_innovation_negg": [0.018002334982156754, -0.013630302622914314, 0.025494003668427467], "cos_apical_negg": [0.022167127579450607, -0.21593934297561646, -0.1491767317056656], "cos_Ac_negg": [0.017906315624713898, -0.013631798326969147, 0.025503316894173622], "r_norm": [3.0101892948150635, 3.587337017059326, 3.8490867614746094], "ldr": {"dL_sdil": -2.548330307006836, "dL_bp": -2.548330307006836, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 15.974457740783691, "cos_r_negg": [0.05967242270708084, -0.1192544624209404, -0.6155177354812622], "cos_innovation_negg": [0.12467913329601288, 0.18524280190467834, 0.41068893671035767], "cos_apical_negg": [0.05967242270708084, -0.1192544624209404, -0.6155176758766174], "cos_Ac_negg": [0.1246727854013443, 0.18524479866027832, 0.4107542335987091], "r_norm": [2.892625093460083, 3.2755002975463867, 3.376058578491211]}, {"step": 1700, "epoch": 3, "train_loss": 11.454575538635254, "cos_r_negg": [0.05775633454322815, -0.1839609146118164, -0.4234389662742615], "cos_innovation_negg": [-0.02099188044667244, -0.01595008373260498, 0.4565528631210327], "cos_apical_negg": [0.05775633454322815, -0.1839609295129776, -0.4234389662742615], "cos_Ac_negg": [-0.021001555025577545, -0.015945378690958023, 0.45654499530792236], "r_norm": [2.5749480724334717, 2.916334629058838, 3.2268168926239014]}, {"step": 1800, "epoch": 3, "train_loss": 12.229915618896484, "cos_r_negg": [0.08983103930950165, -0.10578735172748566, -0.16379764676094055], "cos_innovation_negg": [0.10483919829130173, 0.28883856534957886, 0.44382697343826294], "cos_apical_negg": [0.08983103930950165, -0.10578733682632446, -0.16379764676094055], "cos_Ac_negg": [0.10483229160308838, 0.2888367474079132, 0.44381600618362427], "r_norm": [2.640359878540039, 2.885666608810425, 3.2105512619018555]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.098, "test_loss": 10.5088134765625}, {"step": 1900, "epoch": 4, "train_loss": 17.48843002319336, "cos_r_negg": [0.012519306503236294, -0.4328705668449402, -0.5602304935455322], "cos_innovation_negg": [-0.049148138612508774, 0.28105664253234863, 0.7195155620574951], "cos_apical_negg": [0.012519307434558868, -0.4328705668449402, -0.5602304339408875], "cos_Ac_negg": [-0.04915396124124527, 0.2810572683811188, 0.7195122241973877], "r_norm": [2.3489015102386475, 2.694956064224243, 2.9692583084106445]}, {"step": 2000, "epoch": 4, "train_loss": 9.76813793182373, "cos_r_negg": [-0.018307412043213844, 0.06446070969104767, -0.10387585312128067], "cos_innovation_negg": [-0.0366482250392437, 0.2162434458732605, 0.6224699020385742], "cos_apical_negg": [-0.018307410180568695, 0.06446070969104767, -0.10387587547302246], "cos_Ac_negg": [-0.03666269779205322, 0.21624931693077087, 0.622467041015625], "r_norm": [2.511305332183838, 2.9758949279785156, 3.3254880905151367], "ldr": {"dL_sdil": -1.9794464111328125, "dL_bp": -1.9794464111328125, "ratio": 1.0}}, {"step": 2100, "epoch": 4, "train_loss": 7.482757568359375, "cos_r_negg": [0.03618372604250908, -0.13149124383926392, -0.48567989468574524], "cos_innovation_negg": [0.07447533309459686, 0.06922244280576706, 0.45074909925460815], "cos_apical_negg": [0.03618372604250908, -0.13149124383926392, -0.4856799244880676], "cos_Ac_negg": [0.07447478175163269, 0.06922130286693573, 0.4507486820220947], "r_norm": [2.409442901611328, 2.776766777038574, 3.0350341796875]}, {"step": 2200, "epoch": 4, "train_loss": 13.395278930664062, "cos_r_negg": [0.030296243727207184, -0.2916486859321594, -0.4456755518913269], "cos_innovation_negg": [0.0532541424036026, -0.2333734631538391, 0.8114545345306396], "cos_apical_negg": [0.030296241864562035, -0.2916486859321594, -0.4456755518913269], "cos_Ac_negg": [0.053279198706150055, -0.23337127268314362, 0.81145179271698], "r_norm": [2.322329521179199, 2.8261942863464355, 3.2641398906707764]}, {"step": 2300, "epoch": 4, "train_loss": 9.56633186340332, "cos_r_negg": [-0.0007144114933907986, -0.08392734825611115, -0.31144028902053833], "cos_innovation_negg": [0.02680947631597519, -0.05834192782640457, 0.5731551647186279], "cos_apical_negg": [-0.000714409165084362, -0.08392734825611115, -0.31144025921821594], "cos_Ac_negg": [0.02680298313498497, -0.058338649570941925, 0.5731555223464966], "r_norm": [1.9384572505950928, 2.2706122398376465, 2.636899709701538]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.0892, "test_loss": 11.87872587890625}, {"step": 2400, "epoch": 5, "train_loss": 15.17947769165039, "cos_r_negg": [0.027683284133672714, -0.12447544932365417, -0.5601911544799805], "cos_innovation_negg": [-0.037985533475875854, -0.10906483232975006, 0.2888246178627014], "cos_apical_negg": [0.027683287858963013, -0.12447544187307358, -0.56019127368927], "cos_Ac_negg": [-0.0379890538752079, -0.1090637668967247, 0.28882139921188354], "r_norm": [1.9936455488204956, 2.347970485687256, 2.744025468826294]}, {"step": 2500, "epoch": 5, "train_loss": 12.193578720092773, "cos_r_negg": [-0.0026660682633519173, -0.18455472588539124, -0.259928822517395], "cos_innovation_negg": [-0.03150944411754608, 0.10283806920051575, 0.5210461616516113], "cos_apical_negg": [-0.0026660673320293427, -0.18455474078655243, -0.2599288523197174], "cos_Ac_negg": [-0.03151656687259674, 0.1028411015868187, 0.5210443735122681], "r_norm": [1.9233973026275635, 2.319603443145752, 2.8037376403808594], "ldr": {"dL_sdil": -2.6694507598876953, "dL_bp": -2.6694507598876953, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 13.220319747924805, "cos_r_negg": [0.02900480292737484, -0.2758943438529968, -0.4539209306240082], "cos_innovation_negg": [-0.011245804838836193, -0.12144125998020172, 0.09843893349170685], "cos_apical_negg": [0.02900479920208454, -0.2758943438529968, -0.45392096042633057], "cos_Ac_negg": [-0.011244049295783043, -0.12144289910793304, 0.09843729436397552], "r_norm": [1.8653290271759033, 2.214156150817871, 2.741147518157959]}, {"step": 2700, "epoch": 5, "train_loss": 10.434335708618164, "cos_r_negg": [0.05317149683833122, -0.11676039546728134, -0.3232342302799225], "cos_innovation_negg": [-0.011293860152363777, 0.3770521879196167, 0.7010574340820312], "cos_apical_negg": [0.05317150056362152, -0.11676039546728134, -0.3232342302799225], "cos_Ac_negg": [-0.011303890496492386, 0.37705329060554504, 0.7010572552680969], "r_norm": [1.9065091609954834, 2.2450571060180664, 2.7173304557800293]}, {"step": 2800, "epoch": 5, "train_loss": 8.608712196350098, "cos_r_negg": [0.021214831620454788, -0.22861573100090027, -0.27367472648620605], "cos_innovation_negg": [0.025810478255152702, 0.183097243309021, 0.43575942516326904], "cos_apical_negg": [0.02121482789516449, -0.22861573100090027, -0.27367472648620605], "cos_Ac_negg": [0.02580975368618965, 0.18310371041297913, 0.4357585310935974], "r_norm": [1.9713449478149414, 2.2843751907348633, 2.9697022438049316]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.101, "test_loss": 18.1558404296875}, {"step": 2900, "epoch": 6, "train_loss": 10.255294799804688, "cos_r_negg": [0.03914017975330353, -0.0356130450963974, -0.04506968706846237], "cos_innovation_negg": [-0.03473691642284393, 0.1018490195274353, 0.42403528094291687], "cos_apical_negg": [0.039140187203884125, -0.0356130450963974, -0.045069705694913864], "cos_Ac_negg": [-0.03474500775337219, 0.10183992981910706, 0.4240354597568512], "r_norm": [1.725934624671936, 2.0168943405151367, 2.6580262184143066]}, {"step": 3000, "epoch": 6, "train_loss": 12.970020294189453, "cos_r_negg": [-0.004842756316065788, -0.12890617549419403, -0.6363621950149536], "cos_innovation_negg": [-0.047544751316308975, -0.005340326577425003, 0.28954434394836426], "cos_apical_negg": [-0.004842751659452915, -0.12890617549419403, -0.6363622546195984], "cos_Ac_negg": [-0.0472957044839859, -0.005311688408255577, 0.28955477476119995], "r_norm": [2.0303399562835693, 2.3841023445129395, 3.0937414169311523], "ldr": {"dL_sdil": -4.037936210632324, "dL_bp": -4.037936210632324, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 14.473289489746094, "cos_r_negg": [0.04678209498524666, -0.010735692456364632, -0.17836451530456543], "cos_innovation_negg": [0.042138442397117615, -0.028546959161758423, 0.29168370366096497], "cos_apical_negg": [0.04678209871053696, -0.010735689662396908, -0.17836448550224304], "cos_Ac_negg": [0.042134158313274384, -0.028544962406158447, 0.291683167219162], "r_norm": [1.7289745807647705, 2.0565102100372314, 2.7483131885528564]}, {"step": 3200, "epoch": 6, "train_loss": 9.455717086791992, "cos_r_negg": [0.006103653460741043, -0.08823632448911667, -0.36146461963653564], "cos_innovation_negg": [-0.025493118911981583, -0.25060561299324036, 0.5729857683181763], "cos_apical_negg": [0.0061036525294184685, -0.08823632448911667, -0.36146461963653564], "cos_Ac_negg": [-0.0254963506013155, -0.25061002373695374, 0.5729857683181763], "r_norm": [1.564295768737793, 1.8498930931091309, 2.5144753456115723]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.1135, "test_loss": 8.644243359375}, {"step": 3300, "epoch": 7, "train_loss": 10.686386108398438, "cos_r_negg": [0.007331964559853077, -0.1437041163444519, -0.4923170804977417], "cos_innovation_negg": [-0.017183762043714523, 0.2204362154006958, 0.8977514505386353], "cos_apical_negg": [0.007331966422498226, -0.1437041312456131, -0.4923170804977417], "cos_Ac_negg": [-0.017086660489439964, 0.2204228937625885, 0.8977519273757935], "r_norm": [2.1216511726379395, 2.4493775367736816, 3.102795362472534]}, {"step": 3400, "epoch": 7, "train_loss": 18.421329498291016, "cos_r_negg": [0.046245694160461426, -0.17511767148971558, -0.40243658423423767], "cos_innovation_negg": [0.05551280826330185, 0.4356822371482849, 0.42793047428131104], "cos_apical_negg": [0.04624570161104202, -0.17511765658855438, -0.4024365544319153], "cos_Ac_negg": [0.05310817435383797, 0.43637800216674805, 0.4279768466949463], "r_norm": [1.860713243484497, 2.2496790885925293, 3.145846366882324]}, {"step": 3500, "epoch": 7, "train_loss": 6.89385986328125, "cos_r_negg": [0.06364455074071884, -0.23486125469207764, -0.5209711790084839], "cos_innovation_negg": [0.032999224960803986, -0.09086863696575165, 0.5897722244262695], "cos_apical_negg": [0.06364455074071884, -0.23486125469207764, -0.5209711790084839], "cos_Ac_negg": [0.03298337385058403, -0.09088343381881714, 0.5897735357284546], "r_norm": [1.8219056129455566, 2.113788604736328, 2.887299060821533], "ldr": {"dL_sdil": -2.7791242599487305, "dL_bp": -2.779125213623047, "ratio": 0.9999996568437033}}, {"step": 3600, "epoch": 7, "train_loss": 10.256458282470703, "cos_r_negg": [0.020578347146511078, -0.14359118044376373, -0.3098403215408325], "cos_innovation_negg": [0.06512169539928436, -0.17804598808288574, 0.6493154764175415], "cos_apical_negg": [0.020578349009156227, -0.14359118044376373, -0.3098403811454773], "cos_Ac_negg": [0.06512264907360077, -0.1780502200126648, 0.6493154764175415], "r_norm": [1.506436824798584, 1.770148754119873, 2.5789108276367188]}, {"step": 3700, "epoch": 7, "train_loss": 10.41223430633545, "cos_r_negg": [0.008857296779751778, -0.2126459926366806, -0.5968126058578491], "cos_innovation_negg": [0.04915352910757065, 0.2753782570362091, 0.2443518340587616], "cos_apical_negg": [0.008857302367687225, -0.2126459777355194, -0.5968126654624939], "cos_Ac_negg": [0.04870340973138809, 0.2754344344139099, 0.24440880119800568], "r_norm": [1.762682557106018, 2.0029869079589844, 3.0203115940093994]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.0982, "test_loss": 11.48839501953125}, {"step": 3800, "epoch": 8, "train_loss": 13.540163040161133, "cos_r_negg": [-0.02213909476995468, -0.07909062504768372, -0.6160959005355835], "cos_innovation_negg": [-0.02084731124341488, 0.12267974764108658, 0.3846050500869751], "cos_apical_negg": [-0.02213909476995468, -0.07909061759710312, -0.6160959005355835], "cos_Ac_negg": [-0.020849252119660378, 0.12268407642841339, 0.3846050500869751], "r_norm": [1.7494614124298096, 1.9818449020385742, 3.011733055114746]}, {"step": 3900, "epoch": 8, "train_loss": 9.196820259094238, "cos_r_negg": [-0.0031318701803684235, -0.15781190991401672, -0.21378548443317413], "cos_innovation_negg": [-0.06686866283416748, 0.43560338020324707, 0.5173091888427734], "cos_apical_negg": [-0.003131865058094263, -0.15781190991401672, -0.21378546953201294], "cos_Ac_negg": [-0.06688021868467331, 0.4355728030204773, 0.5173093676567078], "r_norm": [1.6178021430969238, 1.7688393592834473, 2.8990988731384277]}, {"step": 4000, "epoch": 8, "train_loss": 10.564874649047852, "cos_r_negg": [0.013686101883649826, -0.18070998787879944, -0.41389593482017517], "cos_innovation_negg": [0.0857563391327858, -0.07245780527591705, 0.38016122579574585], "cos_apical_negg": [0.013686100021004677, -0.18070998787879944, -0.41389593482017517], "cos_Ac_negg": [0.08574873954057693, -0.07244805991649628, 0.38016101717948914], "r_norm": [1.4692742824554443, 1.6845234632492065, 2.6116912364959717], "ldr": {"dL_sdil": -2.3447189331054688, "dL_bp": -2.344719409942627, "ratio": 0.9999997966335945}}, {"step": 4100, "epoch": 8, "train_loss": 7.10745096206665, "cos_r_negg": [0.037930723279714584, -0.21142877638339996, -0.036002419888973236], "cos_innovation_negg": [-0.021827008575201035, 0.1278681755065918, 0.7639685869216919], "cos_apical_negg": [0.03793072700500488, -0.21142877638339996, -0.03600241243839264], "cos_Ac_negg": [-0.021835312247276306, 0.12787224352359772, 0.7639685869216919], "r_norm": [1.507741928100586, 1.8093645572662354, 2.850492000579834]}, {"step": 4200, "epoch": 8, "train_loss": 14.092636108398438, "cos_r_negg": [0.06115858256816864, -0.2938695549964905, -0.5185893774032593], "cos_innovation_negg": [0.09283484518527985, 0.08690566569566727, 0.6516421437263489], "cos_apical_negg": [0.06115858256816864, -0.2938695549964905, -0.5185893774032593], "cos_Ac_negg": [0.0929117202758789, 0.08681941777467728, 0.6516423225402832], "r_norm": [1.6354327201843262, 1.876108169555664, 2.946805953979492]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.1028, "test_loss": 17.0794158203125}, {"step": 4300, "epoch": 9, "train_loss": 5.8485517501831055, "cos_r_negg": [0.02849375270307064, 0.0641171932220459, 0.05944167077541351], "cos_innovation_negg": [0.08906682580709457, 0.1299668848514557, 0.7520847916603088], "cos_apical_negg": [0.02849375270307064, 0.0641172006726265, 0.059441663324832916], "cos_Ac_negg": [0.08903417736291885, 0.12996794283390045, 0.7520849704742432], "r_norm": [1.6232593059539795, 1.9098446369171143, 3.160547971725464]}, {"step": 4400, "epoch": 9, "train_loss": 11.522689819335938, "cos_r_negg": [0.0014117984101176262, -0.05524075776338577, -0.034860242158174515], "cos_innovation_negg": [-0.03014400601387024, -0.0960920974612236, 0.7208172082901001], "cos_apical_negg": [0.0014117946848273277, -0.055240750312805176, -0.03486024588346481], "cos_Ac_negg": [-0.030146276578307152, -0.09609593451023102, 0.7208172082901001], "r_norm": [1.3885986804962158, 1.6415561437606812, 2.7161850929260254]}, {"step": 4500, "epoch": 9, "train_loss": 7.704092025756836, "cos_r_negg": [0.009524454362690449, 0.025368399918079376, -0.03347320854663849], "cos_innovation_negg": [0.0515335313975811, 0.05315358191728592, 0.4637925624847412], "cos_apical_negg": [0.00952446274459362, 0.025368396192789078, -0.03347321227192879], "cos_Ac_negg": [0.05153655260801315, 0.053153470158576965, 0.4637925326824188], "r_norm": [1.4529674053192139, 1.6545417308807373, 2.9059221744537354], "ldr": {"dL_sdil": -2.081070899963379, "dL_bp": -2.081070899963379, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 6.669431686401367, "cos_r_negg": [0.01777573674917221, -0.09210150688886642, -0.10563072562217712], "cos_innovation_negg": [0.10005220770835876, 0.16834776103496552, 0.3849782347679138], "cos_apical_negg": [0.01777573861181736, -0.09210151433944702, -0.10563073307275772], "cos_Ac_negg": [0.1000543087720871, 0.16835026443004608, 0.3849782347679138], "r_norm": [1.3946471214294434, 1.578935146331787, 2.8077735900878906]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.1135, "test_loss": 9.015954296875}, {"step": 4700, "epoch": 10, "train_loss": 9.554258346557617, "cos_r_negg": [-0.01756708323955536, -0.04851745814085007, -0.002075500786304474], "cos_innovation_negg": [-0.08733142912387848, 0.07131969928741455, 0.6723060607910156], "cos_apical_negg": [-0.01756708323955536, -0.048517461866140366, -0.0020755035802721977], "cos_Ac_negg": [-0.08734530955553055, 0.07131726294755936, 0.6723060607910156], "r_norm": [1.3125298023223877, 1.5057177543640137, 2.6409571170806885]}, {"step": 4800, "epoch": 10, "train_loss": 5.9013471603393555, "cos_r_negg": [0.0754239559173584, -0.05168905481696129, 0.11058440804481506], "cos_innovation_negg": [0.007327036000788212, 0.2709810137748718, 0.7619322538375854], "cos_apical_negg": [0.0754239559173584, -0.05168905854225159, 0.11058438569307327], "cos_Ac_negg": [0.007336634211242199, 0.2709811329841614, 0.7619322538375854], "r_norm": [1.463352918624878, 1.5180680751800537, 2.9792442321777344]}, {"step": 4900, "epoch": 10, "train_loss": 10.935822486877441, "cos_r_negg": [0.0575990304350853, -0.33695322275161743, -0.34385931491851807], "cos_innovation_negg": [0.014982819557189941, 0.09094658493995667, 0.4001518189907074], "cos_apical_negg": [0.0575990229845047, -0.33695322275161743, -0.34385931491851807], "cos_Ac_negg": [0.015041286125779152, 0.09094700217247009, 0.40015166997909546], "r_norm": [1.4888331890106201, 1.616701602935791, 3.0466463565826416]}, {"step": 5000, "epoch": 10, "train_loss": 9.521804809570312, "cos_r_negg": [0.05206938087940216, -0.3930056393146515, -0.5807730555534363], "cos_innovation_negg": [-0.04468301683664322, 0.14414645731449127, 0.5752860903739929], "cos_apical_negg": [0.05206938832998276, -0.3930056393146515, -0.5807730555534363], "cos_Ac_negg": [-0.044713109731674194, 0.14414654672145844, 0.5752860903739929], "r_norm": [1.3562431335449219, 1.4976449012756348, 2.7944326400756836], "ldr": {"dL_sdil": -1.940413475036621, "dL_bp": -1.940413475036621, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 7.874048233032227, "cos_r_negg": [-0.024124640971422195, 0.058301106095314026, 0.07119216024875641], "cos_innovation_negg": [-0.05157509446144104, 0.05782962590456009, 0.5174257159233093], "cos_apical_negg": [-0.024124637246131897, 0.058301106095314026, 0.0711921826004982], "cos_Ac_negg": [-0.05156651884317398, 0.057829566299915314, 0.5174257159233093], "r_norm": [1.3540897369384766, 1.5044630765914917, 2.9336659908294678]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.1009, "test_loss": 11.43567568359375}, {"step": 5200, "epoch": 11, "train_loss": 10.689697265625, "cos_r_negg": [0.019001197069883347, -0.2897270917892456, -0.2654634714126587], "cos_innovation_negg": [-0.05334196239709854, 0.08347544819116592, 0.39443832635879517], "cos_apical_negg": [0.019001193344593048, -0.2897270917892456, -0.2654634714126587], "cos_Ac_negg": [-0.0533544197678566, 0.08347558230161667, 0.39443832635879517], "r_norm": [1.2976155281066895, 1.4617466926574707, 2.853634834289551]}, {"step": 5300, "epoch": 11, "train_loss": 10.256033897399902, "cos_r_negg": [0.013968532904982567, -0.22812309861183167, -0.13791435956954956], "cos_innovation_negg": [-0.05052020028233528, 0.12567764520645142, 0.5747710466384888], "cos_apical_negg": [0.01396853569895029, -0.22812311351299286, -0.13791435956954956], "cos_Ac_negg": [-0.05052119493484497, 0.12567763030529022, 0.5747710466384888], "r_norm": [1.1869971752166748, 1.3122568130493164, 2.6022799015045166]}, {"step": 5400, "epoch": 11, "train_loss": 9.13038444519043, "cos_r_negg": [0.019583577290177345, -0.3641751706600189, -0.526611328125], "cos_innovation_negg": [-0.0005453093908727169, -0.12072335928678513, 0.5536854267120361], "cos_apical_negg": [0.019583577290177345, -0.36417514085769653, -0.5266112685203552], "cos_Ac_negg": [-0.0005325092934072018, -0.12072324752807617, 0.5536854267120361], "r_norm": [1.459100604057312, 1.5429048538208008, 3.1018805503845215]}, {"step": 5500, "epoch": 11, "train_loss": 13.559499740600586, "cos_r_negg": [0.06639349460601807, -0.5439437031745911, -0.5144651532173157], "cos_innovation_negg": [-0.06592432409524918, -0.18451941013336182, 0.861040472984314], "cos_apical_negg": [0.06639350205659866, -0.5439437627792358, -0.5144652128219604], "cos_Ac_negg": [-0.06593867391347885, -0.18451935052871704, 0.8610404133796692], "r_norm": [1.2953298091888428, 1.4854196310043335, 2.7311806678771973], "ldr": {"dL_sdil": -2.1915130615234375, "dL_bp": -2.1915130615234375, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 11.20883560180664, "cos_r_negg": [-0.031799525022506714, -0.22811442613601685, -0.23479770123958588], "cos_innovation_negg": [-0.0690169632434845, -0.0075832391157746315, 0.4777218997478485], "cos_apical_negg": [-0.03179951757192612, -0.22811444103717804, -0.23479770123958588], "cos_Ac_negg": [-0.06903363764286041, -0.0075830332934856415, 0.4777218699455261], "r_norm": [1.340540885925293, 1.5263988971710205, 3.144881248474121]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.0974, "test_loss": 13.76685361328125}, {"step": 5700, "epoch": 12, "train_loss": 10.07309341430664, "cos_r_negg": [0.03777661919593811, -0.4412369728088379, -0.36332398653030396], "cos_innovation_negg": [0.03472915291786194, -0.17701226472854614, 0.8993306756019592], "cos_apical_negg": [0.03777661919593811, -0.4412369728088379, -0.36332398653030396], "cos_Ac_negg": [0.034731801599264145, -0.17701216042041779, 0.899330735206604], "r_norm": [1.3225698471069336, 1.5236485004425049, 2.700747013092041]}, {"step": 5800, "epoch": 12, "train_loss": 12.433907508850098, "cos_r_negg": [-0.003950262442231178, -0.547705352306366, -0.3867746889591217], "cos_innovation_negg": [-0.05912623554468155, -0.19530336558818817, 0.9709994792938232], "cos_apical_negg": [-0.003950268030166626, -0.547705352306366, -0.3867747187614441], "cos_Ac_negg": [-0.05908357352018356, -0.1953035295009613, 0.9709994792938232], "r_norm": [1.3607196807861328, 1.4850249290466309, 3.2782058715820312]}, {"step": 5900, "epoch": 12, "train_loss": 9.572389602661133, "cos_r_negg": [0.0695822685956955, -0.4782870411872864, -0.4824511706829071], "cos_innovation_negg": [-0.0036430712789297104, 0.21301300823688507, 0.6043546795845032], "cos_apical_negg": [0.0695822685956955, -0.478287011384964, -0.4824511408805847], "cos_Ac_negg": [-0.0036581167951226234, 0.21301306784152985, 0.604354739189148], "r_norm": [1.212158441543579, 1.313722848892212, 2.773188352584839]}, {"step": 6000, "epoch": 12, "train_loss": 8.03459644317627, "cos_r_negg": [0.05641988664865494, -0.30625995993614197, -0.32399484515190125], "cos_innovation_negg": [0.07821530103683472, -0.18439939618110657, 0.7435494661331177], "cos_apical_negg": [0.05641988664865494, -0.30625995993614197, -0.32399487495422363], "cos_Ac_negg": [0.07819831371307373, -0.18439938127994537, 0.7435494661331177], "r_norm": [1.2636423110961914, 1.4633649587631226, 2.6897456645965576], "ldr": {"dL_sdil": -1.2584352493286133, "dL_bp": -1.2584352493286133, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.0974, "test_loss": 6.334162353515625}, {"step": 6100, "epoch": 13, "train_loss": 5.32422399520874, "cos_r_negg": [-0.020511440932750702, -0.20042458176612854, -0.1201372966170311], "cos_innovation_negg": [-0.05921708047389984, 0.0025104722008109093, 0.7776116132736206], "cos_apical_negg": [-0.020511429756879807, -0.20042458176612854, -0.1201372966170311], "cos_Ac_negg": [-0.05921370908617973, 0.0025104996748268604, 0.7776116132736206], "r_norm": [1.1844608783721924, 1.339348554611206, 2.631443977355957]}, {"step": 6200, "epoch": 13, "train_loss": 15.338651657104492, "cos_r_negg": [0.006799392402172089, -0.4675729274749756, -0.5238707065582275], "cos_innovation_negg": [0.016532087698578835, 0.22660422325134277, 0.5020425319671631], "cos_apical_negg": [0.006799392402172089, -0.4675729274749756, -0.5238707065582275], "cos_Ac_negg": [0.016515430063009262, 0.22660425305366516, 0.5020426511764526], "r_norm": [1.2123239040374756, 1.321899175643921, 2.875114917755127]}, {"step": 6300, "epoch": 13, "train_loss": 10.865857124328613, "cos_r_negg": [-0.04579303413629532, -0.3763803243637085, -0.2255067229270935], "cos_innovation_negg": [-0.031234746798872948, -0.18446262180805206, 0.718535304069519], "cos_apical_negg": [-0.04579302668571472, -0.3763803243637085, -0.2255067080259323], "cos_Ac_negg": [-0.0312519297003746, -0.18446236848831177, 0.718535304069519], "r_norm": [1.1536757946014404, 1.2023568153381348, 2.787022829055786]}, {"step": 6400, "epoch": 13, "train_loss": 10.130316734313965, "cos_r_negg": [0.014571597799658775, -0.3073262870311737, -0.2725748121738434], "cos_innovation_negg": [-0.0355323888361454, -0.0392596572637558, 0.7970172166824341], "cos_apical_negg": [0.014571597799658775, -0.3073262870311737, -0.2725748121738434], "cos_Ac_negg": [-0.03557465225458145, -0.03925967589020729, 0.7970172166824341], "r_norm": [1.270211100578308, 1.2800111770629883, 3.166908025741577]}, {"step": 6500, "epoch": 13, "train_loss": 10.813982009887695, "cos_r_negg": [0.011883115395903587, -0.5175911784172058, -0.3062574565410614], "cos_innovation_negg": [-0.005955313332378864, -0.013851936906576157, 0.91746985912323], "cos_apical_negg": [0.011883115395903587, -0.5175911784172058, -0.306257426738739], "cos_Ac_negg": [-0.005974724888801575, -0.013851940631866455, 0.91746985912323], "r_norm": [1.1609448194503784, 1.2118589878082275, 2.9055025577545166], "ldr": {"dL_sdil": -1.5699901580810547, "dL_bp": -1.5699901580810547, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.1032, "test_loss": 11.97496240234375}, {"step": 6600, "epoch": 14, "train_loss": 10.166065216064453, "cos_r_negg": [0.0740952342748642, -0.42438608407974243, -0.5446914434432983], "cos_innovation_negg": [0.007209917530417442, -0.049284063279628754, 0.5896706581115723], "cos_apical_negg": [0.0740952342748642, -0.42438608407974243, -0.5446915030479431], "cos_Ac_negg": [0.007204357534646988, -0.04928404465317726, 0.589670717716217], "r_norm": [1.1379551887512207, 1.2085778713226318, 2.7993202209472656]}, {"step": 6700, "epoch": 14, "train_loss": 12.796271324157715, "cos_r_negg": [0.04596796631813049, -0.4727451801300049, -0.2784838080406189], "cos_innovation_negg": [-0.15685194730758667, -0.2215430736541748, 0.6244687438011169], "cos_apical_negg": [0.04596797004342079, -0.4727451801300049, -0.2784837782382965], "cos_Ac_negg": [-0.15674909949302673, -0.22154322266578674, 0.6244687438011169], "r_norm": [1.2866442203521729, 1.2906875610351562, 3.3445470333099365]}, {"step": 6800, "epoch": 14, "train_loss": 10.457527160644531, "cos_r_negg": [0.0034635011106729507, -0.33966535329818726, -0.3185226321220398], "cos_innovation_negg": [-0.015793822705745697, 0.06659657508134842, 0.4613923132419586], "cos_apical_negg": [0.003463505767285824, -0.33966535329818726, -0.3185226619243622], "cos_Ac_negg": [-0.01584102213382721, 0.06659658253192902, 0.4613923132419586], "r_norm": [1.164564847946167, 1.3001453876495361, 3.0376219749450684]}, {"step": 6900, "epoch": 14, "train_loss": 5.674578666687012, "cos_r_negg": [0.06117851287126541, -0.4614144265651703, -0.3935377895832062], "cos_innovation_negg": [0.08230698853731155, 0.23889562487602234, 0.44478482007980347], "cos_apical_negg": [0.06117851659655571, -0.4614144265651703, -0.3935377597808838], "cos_Ac_negg": [0.08229748904705048, 0.23889559507369995, 0.4447847902774811], "r_norm": [1.207074522972107, 1.301612138748169, 3.0882766246795654]}, {"step": 7000, "epoch": 14, "train_loss": 10.910627365112305, "cos_r_negg": [0.03731546550989151, -0.37948036193847656, -0.3181638717651367], "cos_innovation_negg": [-0.02078831009566784, 0.196212038397789, 0.4585264325141907], "cos_apical_negg": [0.03731546178460121, -0.37948036193847656, -0.3181639015674591], "cos_Ac_negg": [-0.02077524922788143, 0.1962120234966278, 0.45852649211883545], "r_norm": [1.1883717775344849, 1.262464165687561, 3.0732603073120117], "ldr": {"dL_sdil": -1.7473487854003906, "dL_bp": -1.7473487854003906, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1028, "test_loss": 17.5008279296875}], "final": {"test_acc": 0.1028, "test_loss": 17.5008279296875, "wall_s": 27.01093864440918, "cos_r_negg": [0.11227206885814667, -0.525943398475647, -0.6144851446151733], "cos_innovation_negg": [-0.17259691655635834, -0.044658537954092026, 0.6417496800422668], "cos_apical_negg": [0.11227206885814667, -0.5259433388710022, -0.6144851446151733], "cos_Ac_negg": [-0.1726006418466568, -0.04465854912996292, 0.6417496800422668]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_matched_s2.json b/results/nuis_ctrl_v1_rho0p5_matched_s2.json new file mode 100644 index 0000000..abeed39 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_matched_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_matched_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.011090951040387154, 0.025726400315761566, 0.00020627662888728082], "cos_innovation_negg": [0.014441920444369316, 0.03731844574213028, 0.002671795664355159], "cos_apical_negg": [0.011090951040387154, 0.025726400315761566, 0.00020627668709494174], "cos_Ac_negg": [0.0144441407173872, 0.03731893002986908, 0.0026724948547780514], "r_norm": [4.7590131759643555, 4.78401517868042, 4.868775367736816], "ldr": {"dL_sdil": -0.048075199127197266, "dL_bp": -0.20142126083374023, "ratio": 0.2386798639239982}}, {"step": 100, "epoch": 0, "train_loss": 13.749383926391602, "cos_r_negg": [-0.06069348752498627, -0.1737738847732544, -0.669389009475708], "cos_innovation_negg": [-0.06193709373474121, -0.13967296481132507, -0.42725348472595215], "cos_apical_negg": [-0.06069348007440567, -0.1737738847732544, -0.669389009475708], "cos_Ac_negg": [-0.06194448098540306, -0.13965845108032227, -0.4273082911968231], "r_norm": [5.48262882232666, 5.59251594543457, 5.765050888061523]}, {"step": 200, "epoch": 0, "train_loss": 8.081483840942383, "cos_r_negg": [-0.01503482274711132, 0.09153439104557037, -0.36269283294677734], "cos_innovation_negg": [-0.04903820902109146, 0.20579181611537933, 0.09409314393997192], "cos_apical_negg": [-0.015034825541079044, 0.09153439104557037, -0.36269286274909973], "cos_Ac_negg": [-0.04876821115612984, 0.2059166133403778, 0.09408814460039139], "r_norm": [5.215045928955078, 5.352688789367676, 5.3576178550720215]}, {"step": 300, "epoch": 0, "train_loss": 10.916276931762695, "cos_r_negg": [0.04431144893169403, -0.09456641227006912, -0.3985884487628937], "cos_innovation_negg": [0.035879217088222504, -0.10038431733846664, -0.2997305989265442], "cos_apical_negg": [0.04431144893169403, -0.09456641972064972, -0.3985884487628937], "cos_Ac_negg": [0.037416793406009674, -0.10048316419124603, -0.29465004801750183], "r_norm": [5.2624921798706055, 5.320481300354004, 5.5306878089904785]}, {"step": 400, "epoch": 0, "train_loss": 8.295561790466309, "cos_r_negg": [-0.04517912119626999, -0.07375495880842209, -0.5778710246086121], "cos_innovation_negg": [-0.05381634086370468, -0.051716968417167664, -0.5813968777656555], "cos_apical_negg": [-0.04517911374568939, -0.07375496625900269, -0.5778710246086121], "cos_Ac_negg": [-0.05371318757534027, -0.05170571058988571, -0.581224262714386], "r_norm": [5.110396385192871, 5.392022132873535, 5.6484293937683105]}, {"epoch_end": 0, "step": 469, "test_acc": 0.1032, "test_loss": 12.5786658203125}, {"step": 500, "epoch": 1, "train_loss": 10.815725326538086, "cos_r_negg": [0.06933513283729553, -0.012699754908680916, -0.37486469745635986], "cos_innovation_negg": [0.08232143521308899, 0.38614118099212646, 0.048744603991508484], "cos_apical_negg": [0.06933513283729553, -0.01269975584000349, -0.3748646676540375], "cos_Ac_negg": [0.08280842006206512, 0.3862530589103699, 0.04927188158035278], "r_norm": [4.390542030334473, 4.629734039306641, 4.737148284912109], "ldr": {"dL_sdil": -2.896068572998047, "dL_bp": -2.896068572998047, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 18.51286506652832, "cos_r_negg": [0.02655915543437004, -0.06162973493337631, -0.308250367641449], "cos_innovation_negg": [0.004128217697143555, 0.22477003931999207, 0.062896728515625], "cos_apical_negg": [0.026559149846434593, -0.06162973493337631, -0.30825039744377136], "cos_Ac_negg": [0.0041070356965065, 0.22476521134376526, 0.06287334114313126], "r_norm": [4.20107889175415, 4.465005874633789, 4.517696380615234]}, {"step": 700, "epoch": 1, "train_loss": 12.219400405883789, "cos_r_negg": [-0.0052885133773088455, 0.03356270492076874, -0.2862797975540161], "cos_innovation_negg": [-0.003695674240589142, 0.11314187943935394, -0.10860784351825714], "cos_apical_negg": [-0.005288518033921719, 0.03356270492076874, -0.28627973794937134], "cos_Ac_negg": [-0.003703060559928417, 0.1131385862827301, -0.10861226916313171], "r_norm": [3.564338207244873, 3.821251630783081, 4.026392459869385]}, {"step": 800, "epoch": 1, "train_loss": 8.51838493347168, "cos_r_negg": [-0.0698319524526596, -0.10171648114919662, -0.5155225992202759], "cos_innovation_negg": [-0.11639899760484695, -0.19930781424045563, -0.3248152732849121], "cos_apical_negg": [-0.0698319599032402, -0.10171647369861603, -0.5155225992202759], "cos_Ac_negg": [-0.11640596389770508, -0.19930925965309143, -0.32483217120170593], "r_norm": [3.8340210914611816, 4.210657119750977, 4.495687961578369]}, {"step": 900, "epoch": 1, "train_loss": 9.66305923461914, "cos_r_negg": [-0.024009792134165764, 0.11030104011297226, -0.34016501903533936], "cos_innovation_negg": [0.0019780248403549194, 0.15581965446472168, 0.026185858994722366], "cos_apical_negg": [-0.024009792134165764, 0.11030103266239166, -0.34016501903533936], "cos_Ac_negg": [0.0015606284141540527, 0.1554984748363495, 0.027093322947621346], "r_norm": [4.037374019622803, 4.38667106628418, 4.539052963256836]}, {"epoch_end": 1, "step": 938, "test_acc": 0.101, "test_loss": 6.413751806640625}, {"step": 1000, "epoch": 2, "train_loss": 10.465387344360352, "cos_r_negg": [-0.01651780866086483, -0.10332629084587097, -0.35107189416885376], "cos_innovation_negg": [-0.039348918944597244, -0.02454802766442299, 0.22929388284683228], "cos_apical_negg": [-0.01651781052350998, -0.10332628339529037, -0.35107189416885376], "cos_Ac_negg": [-0.03935210406780243, -0.024546723812818527, 0.22928884625434875], "r_norm": [3.302873373031616, 3.640159845352173, 3.8219363689422607], "ldr": {"dL_sdil": -1.7480230331420898, "dL_bp": -1.7480230331420898, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 9.852148056030273, "cos_r_negg": [-0.05742596834897995, -0.5740408301353455, -0.5695698857307434], "cos_innovation_negg": [-0.16072702407836914, -0.5434074401855469, -0.5429023504257202], "cos_apical_negg": [-0.05742596462368965, -0.5740408301353455, -0.5695699453353882], "cos_Ac_negg": [-0.15696129202842712, -0.5434538722038269, -0.5419536232948303], "r_norm": [3.7044289112091064, 4.175427436828613, 4.482939720153809]}, {"step": 1200, "epoch": 2, "train_loss": 11.632671356201172, "cos_r_negg": [0.0008259671740233898, -0.12051194161176682, -0.3023558259010315], "cos_innovation_negg": [-0.029405130073428154, 0.07660876959562302, -0.09181950986385345], "cos_apical_negg": [0.0008259639143943787, -0.12051194161176682, -0.3023558259010315], "cos_Ac_negg": [-0.029412303119897842, 0.0766005888581276, -0.09183229506015778], "r_norm": [3.080683708190918, 3.521299362182617, 3.684753894805908]}, {"step": 1300, "epoch": 2, "train_loss": 18.60356903076172, "cos_r_negg": [-0.020029690116643906, -0.1791384369134903, -0.3576328158378601], "cos_innovation_negg": [-0.06431147456169128, -0.059316959232091904, -0.30526161193847656], "cos_apical_negg": [-0.020029684528708458, -0.1791384369134903, -0.3576328158378601], "cos_Ac_negg": [-0.06431614607572556, -0.059323206543922424, -0.30526793003082275], "r_norm": [2.9527125358581543, 3.281538963317871, 3.47605562210083]}, {"step": 1400, "epoch": 2, "train_loss": 7.903416633605957, "cos_r_negg": [0.02491527423262596, -0.09285008907318115, -0.23588930070400238], "cos_innovation_negg": [0.007880967110395432, 0.1510201394557953, 0.029834698885679245], "cos_apical_negg": [0.024915270507335663, -0.09285008162260056, -0.23588931560516357], "cos_Ac_negg": [0.007872804068028927, 0.15101441740989685, 0.029831252992153168], "r_norm": [2.806915760040283, 3.1352176666259766, 3.325784206390381]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.1028, "test_loss": 7.140902880859375}, {"step": 1500, "epoch": 3, "train_loss": 13.094808578491211, "cos_r_negg": [-0.010727480985224247, -0.03384358435869217, -0.24551962316036224], "cos_innovation_negg": [0.028976738452911377, 0.1297769546508789, -0.1385250836610794], "cos_apical_negg": [-0.010727478191256523, -0.03384358063340187, -0.24551963806152344], "cos_Ac_negg": [0.028960218653082848, 0.12975086271762848, -0.138573557138443], "r_norm": [2.9644765853881836, 3.4506678581237793, 3.6913342475891113], "ldr": {"dL_sdil": -2.748541831970215, "dL_bp": -2.748541831970215, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 11.407716751098633, "cos_r_negg": [0.0029534650966525078, -0.13059940934181213, -0.39221540093421936], "cos_innovation_negg": [-0.008961060084402561, -0.07262174785137177, -0.08495937287807465], "cos_apical_negg": [0.002953464165329933, -0.13059940934181213, -0.39221540093421936], "cos_Ac_negg": [-0.008964388631284237, -0.07263034582138062, -0.0849585011601448], "r_norm": [2.7864620685577393, 3.0795888900756836, 3.407869338989258]}, {"step": 1700, "epoch": 3, "train_loss": 12.889041900634766, "cos_r_negg": [0.021230295300483704, -0.04711591452360153, -0.34981569647789], "cos_innovation_negg": [-0.005318745970726013, 0.2536885440349579, 0.3438206911087036], "cos_apical_negg": [0.021230291575193405, -0.047115910798311234, -0.34981569647789], "cos_Ac_negg": [-0.005328552797436714, 0.2536941170692444, 0.34381788969039917], "r_norm": [2.34494948387146, 2.7055747509002686, 3.0481817722320557]}, {"step": 1800, "epoch": 3, "train_loss": 8.638145446777344, "cos_r_negg": [0.007142039015889168, 0.07915806770324707, -0.38720494508743286], "cos_innovation_negg": [0.01611199975013733, 0.16419294476509094, 0.1595611870288849], "cos_apical_negg": [0.007142041809856892, 0.07915806025266647, -0.3872049152851105], "cos_Ac_negg": [0.016103491187095642, 0.16418156027793884, 0.15955860912799835], "r_norm": [2.262852668762207, 2.5704822540283203, 2.931986093521118]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.0892, "test_loss": 9.45167685546875}, {"step": 1900, "epoch": 4, "train_loss": 13.730608940124512, "cos_r_negg": [0.0005577374249696732, 0.15166649222373962, -0.21859177947044373], "cos_innovation_negg": [-0.07061313092708588, -0.023223944008350372, 0.315675288438797], "cos_apical_negg": [0.000557740218937397, 0.15166650712490082, -0.21859177947044373], "cos_Ac_negg": [-0.07062053680419922, -0.02324461191892624, 0.3156723082065582], "r_norm": [2.267183780670166, 2.598691463470459, 2.944567918777466]}, {"step": 2000, "epoch": 4, "train_loss": 8.351669311523438, "cos_r_negg": [0.013370946049690247, 0.08591698855161667, -0.09656865149736404], "cos_innovation_negg": [0.012765605002641678, 0.007325336337089539, 0.10097616910934448], "cos_apical_negg": [0.013370947912335396, 0.08591698855161667, -0.09656865149736404], "cos_Ac_negg": [0.012758932076394558, 0.007232479751110077, 0.10097526013851166], "r_norm": [2.7298779487609863, 3.0578556060791016, 3.465477228164673], "ldr": {"dL_sdil": -2.580012321472168, "dL_bp": -2.580012321472168, "ratio": 1.0}}, {"step": 2100, "epoch": 4, "train_loss": 14.770027160644531, "cos_r_negg": [0.04216175526380539, 0.2650410532951355, -0.27206388115882874], "cos_innovation_negg": [-0.07130154967308044, -0.11229027062654495, 0.295823872089386], "cos_apical_negg": [0.04216175526380539, 0.2650410532951355, -0.27206388115882874], "cos_Ac_negg": [-0.07134024798870087, -0.11233744770288467, 0.29582515358924866], "r_norm": [2.488823890686035, 2.9092416763305664, 3.3535714149475098]}, {"step": 2200, "epoch": 4, "train_loss": 7.821073532104492, "cos_r_negg": [0.021576756611466408, 0.17615556716918945, -0.24756211042404175], "cos_innovation_negg": [-0.02474445104598999, -0.015060428529977798, 0.37821513414382935], "cos_apical_negg": [0.02157675288617611, 0.17615556716918945, -0.24756211042404175], "cos_Ac_negg": [-0.024753617122769356, -0.015091238543391228, 0.3782138228416443], "r_norm": [2.1608619689941406, 2.5137405395507812, 2.9557275772094727]}, {"step": 2300, "epoch": 4, "train_loss": 9.868595123291016, "cos_r_negg": [0.04077039659023285, 0.15795400738716125, -0.2698860466480255], "cos_innovation_negg": [-0.009063296020030975, 0.04225557669997215, 0.5083180665969849], "cos_apical_negg": [0.04077039286494255, 0.15795400738716125, -0.2698860764503479], "cos_Ac_negg": [-0.009071932174265385, 0.04223257303237915, 0.5083172917366028], "r_norm": [2.0047760009765625, 2.3676886558532715, 2.81459379196167]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.1009, "test_loss": 10.667335546875}, {"step": 2400, "epoch": 5, "train_loss": 9.581924438476562, "cos_r_negg": [0.02227725461125374, -0.000967247411608696, -0.18699289858341217], "cos_innovation_negg": [0.005947086960077286, 0.033195123076438904, 0.4070398807525635], "cos_apical_negg": [0.02227725461125374, -0.0009672511368989944, -0.18699291348457336], "cos_Ac_negg": [0.005943072959780693, 0.033200476318597794, 0.4070394039154053], "r_norm": [1.8775229454040527, 2.1876232624053955, 2.6086208820343018]}, {"step": 2500, "epoch": 5, "train_loss": 11.010083198547363, "cos_r_negg": [0.02597559429705143, -0.1484498679637909, -0.18701326847076416], "cos_innovation_negg": [0.016522765159606934, 0.05804567039012909, 0.5799378752708435], "cos_apical_negg": [0.02597559243440628, -0.1484498679637909, -0.18701323866844177], "cos_Ac_negg": [0.01652521640062332, 0.058053962886333466, 0.5799371004104614], "r_norm": [2.1859211921691895, 2.58659291267395, 3.2594997882843018], "ldr": {"dL_sdil": -1.7019481658935547, "dL_bp": -1.7019481658935547, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 10.269783020019531, "cos_r_negg": [0.020790455862879753, 0.1781562864780426, -0.36666935682296753], "cos_innovation_negg": [-0.05306564271450043, -0.08230644464492798, 0.5109385251998901], "cos_apical_negg": [0.020790457725524902, 0.1781562864780426, -0.36666932702064514], "cos_Ac_negg": [-0.053070444613695145, -0.08233281970024109, 0.5109382271766663], "r_norm": [2.1497342586517334, 2.57283091545105, 3.1282076835632324]}, {"step": 2700, "epoch": 5, "train_loss": 13.708678245544434, "cos_r_negg": [0.027477143332362175, -0.10665540397167206, -0.1515713334083557], "cos_innovation_negg": [-0.027567919343709946, 0.1947902888059616, 0.42204025387763977], "cos_apical_negg": [0.027477141469717026, -0.10665539652109146, -0.15157130360603333], "cos_Ac_negg": [-0.02767670899629593, 0.19457130134105682, 0.42204177379608154], "r_norm": [2.054931402206421, 2.523709297180176, 3.128479480743408]}, {"step": 2800, "epoch": 5, "train_loss": 7.940017223358154, "cos_r_negg": [0.007737388834357262, 0.11594288051128387, -0.10210414230823517], "cos_innovation_negg": [0.002186122117564082, -0.04148714989423752, 0.4972446858882904], "cos_apical_negg": [0.007737389765679836, 0.11594288051128387, -0.10210414975881577], "cos_Ac_negg": [0.0022293662186712027, -0.04156547039747238, 0.4972487688064575], "r_norm": [2.095334053039551, 2.478428840637207, 3.3034839630126953]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.0958, "test_loss": 12.20820458984375}, {"step": 2900, "epoch": 6, "train_loss": 13.64776611328125, "cos_r_negg": [-0.01721622422337532, 0.036793678998947144, -0.23631972074508667], "cos_innovation_negg": [-0.18655455112457275, -0.15716393291950226, 0.8383230566978455], "cos_apical_negg": [-0.01721622422337532, 0.036793675273656845, -0.23631970584392548], "cos_Ac_negg": [-0.18662171065807343, -0.15739336609840393, 0.8383208513259888], "r_norm": [1.899627685546875, 2.1564974784851074, 3.122950553894043]}, {"step": 3000, "epoch": 6, "train_loss": 14.124244689941406, "cos_r_negg": [-0.012078888714313507, 0.06843101978302002, -0.09456993639469147], "cos_innovation_negg": [-0.0013451261911541224, 0.03656385466456413, 0.6377192735671997], "cos_apical_negg": [-0.012078889645636082, 0.06843102723360062, -0.09456993639469147], "cos_Ac_negg": [-0.0012630023993551731, 0.03625781834125519, 0.6377173662185669], "r_norm": [2.0118227005004883, 2.373253583908081, 3.122997522354126], "ldr": {"dL_sdil": -3.340550422668457, "dL_bp": -3.340550422668457, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 14.035632133483887, "cos_r_negg": [0.024584826081991196, 0.11633805185556412, -0.19864338636398315], "cos_innovation_negg": [-0.07068439573049545, -0.35994952917099, 0.49545717239379883], "cos_apical_negg": [0.024584826081991196, 0.11633804440498352, -0.19864337146282196], "cos_Ac_negg": [-0.07083926349878311, -0.3603760898113251, 0.49546244740486145], "r_norm": [1.9514398574829102, 2.3018431663513184, 3.141934871673584]}, {"step": 3200, "epoch": 6, "train_loss": 10.850109100341797, "cos_r_negg": [0.03840970620512962, 0.11513577401638031, -0.06302927434444427], "cos_innovation_negg": [-0.06367391347885132, -0.4973927438259125, 0.6091084480285645], "cos_apical_negg": [0.038409702479839325, 0.11513577401638031, -0.06302927434444427], "cos_Ac_negg": [-0.06368598341941833, -0.49741876125335693, 0.6091085076332092], "r_norm": [1.8815324306488037, 2.221442937850952, 3.0810980796813965]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.098, "test_loss": 6.482045166015625}, {"step": 3300, "epoch": 7, "train_loss": 7.036479949951172, "cos_r_negg": [-0.0023361407220363617, 0.00536811538040638, -0.11374463140964508], "cos_innovation_negg": [-0.11767683923244476, -0.386144757270813, 0.42218083143234253], "cos_apical_negg": [-0.002336142584681511, 0.00536811538040638, -0.11374462395906448], "cos_Ac_negg": [-0.11768122762441635, -0.38615304231643677, 0.42218074202537537], "r_norm": [1.635523796081543, 1.9011471271514893, 2.6950578689575195]}, {"step": 3400, "epoch": 7, "train_loss": 12.428495407104492, "cos_r_negg": [0.03680352494120598, -0.05252557992935181, -0.07088369131088257], "cos_innovation_negg": [0.0005259551107883453, -0.0348551906645298, 0.4388832449913025], "cos_apical_negg": [0.03680352866649628, -0.05252557992935181, -0.07088368386030197], "cos_Ac_negg": [0.0005234861746430397, -0.03483334556221962, 0.4388831853866577], "r_norm": [1.6723530292510986, 1.8868075609207153, 2.7610669136047363]}, {"step": 3500, "epoch": 7, "train_loss": 7.551629066467285, "cos_r_negg": [0.03003775142133236, 0.1466604322195053, -0.16923099756240845], "cos_innovation_negg": [-0.10296313464641571, -0.11406275629997253, 0.7206891775131226], "cos_apical_negg": [0.03003775328397751, 0.1466604322195053, -0.16923099756240845], "cos_Ac_negg": [-0.10293620079755783, -0.11393731832504272, 0.7206892967224121], "r_norm": [1.7951250076293945, 2.130587100982666, 3.078709602355957], "ldr": {"dL_sdil": -2.0463719367980957, "dL_bp": -2.0463719367980957, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 12.962769508361816, "cos_r_negg": [0.03888365626335144, 0.10250996053218842, -0.1462218165397644], "cos_innovation_negg": [-0.0747726708650589, -0.027483753859996796, 0.7973899841308594], "cos_apical_negg": [0.03888365626335144, 0.10250996798276901, -0.1462218165397644], "cos_Ac_negg": [-0.07424530386924744, -0.026773571968078613, 0.7973881959915161], "r_norm": [1.7965991497039795, 2.1313059329986572, 3.091806411743164]}, {"step": 3700, "epoch": 7, "train_loss": 14.01077651977539, "cos_r_negg": [-0.009441770613193512, 0.1108122169971466, -0.18837007880210876], "cos_innovation_negg": [-0.10919851064682007, 0.08047181367874146, 0.594987154006958], "cos_apical_negg": [-0.009441775269806385, 0.11081220209598541, -0.18837006390094757], "cos_Ac_negg": [-0.11039337515830994, 0.08037280291318893, 0.5949859619140625], "r_norm": [1.7692174911499023, 2.090259552001953, 3.0890002250671387]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.1032, "test_loss": 11.49681220703125}, {"step": 3800, "epoch": 8, "train_loss": 17.658218383789062, "cos_r_negg": [0.0026380959898233414, 0.05858161672949791, -0.4677460193634033], "cos_innovation_negg": [-0.09060388803482056, -0.17893129587173462, -0.051720041781663895], "cos_apical_negg": [0.0026380917988717556, 0.05858161672949791, -0.46774598956108093], "cos_Ac_negg": [-0.0903635174036026, -0.1787409484386444, -0.051720112562179565], "r_norm": [1.8427480459213257, 2.224264621734619, 3.3894171714782715]}, {"step": 3900, "epoch": 8, "train_loss": 9.520259857177734, "cos_r_negg": [0.014783011749386787, 0.1320585161447525, -0.05454879254102707], "cos_innovation_negg": [-0.08465883880853653, -0.3378528952598572, 0.5729187726974487], "cos_apical_negg": [0.014783008024096489, 0.1320585161447525, -0.05454879254102707], "cos_Ac_negg": [-0.08466114103794098, -0.3378331661224365, 0.5729187726974487], "r_norm": [1.3726091384887695, 1.5964603424072266, 2.521697521209717]}, {"step": 4000, "epoch": 8, "train_loss": 12.379359245300293, "cos_r_negg": [7.378868758678436e-06, 0.15252581238746643, 0.11726585030555725], "cos_innovation_negg": [-0.0658871978521347, -0.39697521924972534, 0.7503252029418945], "cos_apical_negg": [7.373746484518051e-06, 0.15252581238746643, 0.11726585030555725], "cos_Ac_negg": [-0.06589606404304504, -0.39698731899261475, 0.7503252625465393], "r_norm": [1.5179842710494995, 1.8182547092437744, 2.883347749710083], "ldr": {"dL_sdil": -1.7692031860351562, "dL_bp": -1.7692031860351562, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 15.944133758544922, "cos_r_negg": [0.008959287777543068, 0.25218236446380615, -0.25705721974372864], "cos_innovation_negg": [-0.11119349300861359, -0.21380172669887543, 0.7379177808761597], "cos_apical_negg": [0.008959285914897919, 0.25218236446380615, -0.25705721974372864], "cos_Ac_negg": [-0.1111975908279419, -0.21380865573883057, 0.7379177808761597], "r_norm": [1.466658115386963, 1.675628900527954, 2.7169625759124756]}, {"step": 4200, "epoch": 8, "train_loss": 6.961349964141846, "cos_r_negg": [-0.024487899616360664, 0.1034964770078659, -0.14107125997543335], "cos_innovation_negg": [-0.033561818301677704, -0.347234308719635, 0.5381101965904236], "cos_apical_negg": [-0.024487897753715515, 0.1034964844584465, -0.14107125997543335], "cos_Ac_negg": [-0.03355357050895691, -0.34724748134613037, 0.5381101965904236], "r_norm": [1.4372986555099487, 1.6103625297546387, 2.709838628768921]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.101, "test_loss": 10.114706640625}, {"step": 4300, "epoch": 9, "train_loss": 9.408743858337402, "cos_r_negg": [0.006029724143445492, -0.01822853460907936, 0.05139956995844841], "cos_innovation_negg": [-0.0064884088933467865, -0.11361739039421082, 0.7829785346984863], "cos_apical_negg": [0.006029726937413216, -0.01822853460907936, 0.051399555057287216], "cos_Ac_negg": [-0.006488284096121788, -0.11362703144550323, 0.7829785346984863], "r_norm": [1.5703096389770508, 1.802883267402649, 3.110020160675049]}, {"step": 4400, "epoch": 9, "train_loss": 11.432577133178711, "cos_r_negg": [0.020391374826431274, -0.2080906629562378, -0.07450072467327118], "cos_innovation_negg": [-0.08861686289310455, 0.21988457441329956, 0.5790855884552002], "cos_apical_negg": [0.020391378551721573, -0.2080906629562378, -0.07450072467327118], "cos_Ac_negg": [-0.08863478153944016, 0.21981927752494812, 0.5790855884552002], "r_norm": [1.3760261535644531, 1.6030185222625732, 2.736809730529785]}, {"step": 4500, "epoch": 9, "train_loss": 5.617181301116943, "cos_r_negg": [-0.0007361690513789654, -0.15166759490966797, -0.17891374230384827], "cos_innovation_negg": [-0.19424927234649658, -0.2667822241783142, 0.803899884223938], "cos_apical_negg": [-0.0007361695170402527, -0.15166759490966797, -0.17891377210617065], "cos_Ac_negg": [-0.1942540407180786, -0.2667856812477112, 0.803899884223938], "r_norm": [1.474501371383667, 1.684433937072754, 2.853506088256836], "ldr": {"dL_sdil": -2.0925254821777344, "dL_bp": -2.0925254821777344, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 8.082502365112305, "cos_r_negg": [-0.008451728150248528, -0.19127199053764343, -0.21168625354766846], "cos_innovation_negg": [-0.19080699980258942, -0.23531076312065125, 0.8365362882614136], "cos_apical_negg": [-0.008451730012893677, -0.19127196073532104, -0.21168623864650726], "cos_Ac_negg": [-0.19081343710422516, -0.23532578349113464, 0.8365359902381897], "r_norm": [1.56576669216156, 1.8033888339996338, 3.051302909851074]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.0958, "test_loss": 14.89981572265625}, {"step": 4700, "epoch": 10, "train_loss": 16.484477996826172, "cos_r_negg": [0.040798772126436234, -0.2947322130203247, 0.10770407319068909], "cos_innovation_negg": [-0.010072074830532074, 0.14773491024971008, 0.7842288017272949], "cos_apical_negg": [0.040798768401145935, -0.2947322130203247, 0.10770407319068909], "cos_Ac_negg": [-0.010081281885504723, 0.14774559438228607, 0.7842288017272949], "r_norm": [1.4422521591186523, 1.6391626596450806, 2.9152655601501465]}, {"step": 4800, "epoch": 10, "train_loss": 9.067924499511719, "cos_r_negg": [0.043948542326688766, -0.48025381565093994, -0.07765369117259979], "cos_innovation_negg": [0.00520036555826664, 0.010058056563138962, 0.734198808670044], "cos_apical_negg": [0.04394853487610817, -0.48025384545326233, -0.0776536613702774], "cos_Ac_negg": [0.00518964696675539, 0.010061800479888916, 0.7341989278793335], "r_norm": [1.3999645709991455, 1.5117855072021484, 2.822413444519043]}, {"step": 4900, "epoch": 10, "train_loss": 12.776921272277832, "cos_r_negg": [0.003369126934558153, -0.29159772396087646, 0.03556031733751297], "cos_innovation_negg": [0.028120510280132294, 0.16014620661735535, 0.6991342306137085], "cos_apical_negg": [0.003369123674929142, -0.29159772396087646, 0.035560302436351776], "cos_Ac_negg": [0.02813071943819523, 0.16014933586120605, 0.6991342306137085], "r_norm": [1.421663522720337, 1.6136540174484253, 2.9729647636413574]}, {"step": 5000, "epoch": 10, "train_loss": 6.876192569732666, "cos_r_negg": [0.04532808065414429, -0.4633644223213196, -0.12176903337240219], "cos_innovation_negg": [0.06101654842495918, -0.06697016954421997, 0.4763108193874359], "cos_apical_negg": [0.04532808065414429, -0.4633644223213196, -0.1217690259218216], "cos_Ac_negg": [0.06105751544237137, -0.06696870177984238, 0.4763107895851135], "r_norm": [1.4727303981781006, 1.7366913557052612, 3.1547532081604004], "ldr": {"dL_sdil": -2.3771209716796875, "dL_bp": -2.3771209716796875, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 9.344998359680176, "cos_r_negg": [0.01690490171313286, -0.40480029582977295, -0.1620321273803711], "cos_innovation_negg": [0.13396157324314117, 0.002423042431473732, 0.6157233715057373], "cos_apical_negg": [0.01690489985048771, -0.40480029582977295, -0.1620321422815323], "cos_Ac_negg": [0.13395491242408752, 0.002431066706776619, 0.6157233715057373], "r_norm": [1.4588286876678467, 1.5729455947875977, 3.0707128047943115]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.0982, "test_loss": 10.66721640625}, {"step": 5200, "epoch": 11, "train_loss": 18.75600242614746, "cos_r_negg": [0.028330041095614433, -0.3266843259334564, 0.13659635186195374], "cos_innovation_negg": [-0.0002497900277376175, 0.02469361573457718, 0.6492017507553101], "cos_apical_negg": [0.028330039232969284, -0.3266843259334564, 0.13659633696079254], "cos_Ac_negg": [-0.0002652667462825775, 0.024695856496691704, 0.6492017507553101], "r_norm": [1.3368139266967773, 1.5819995403289795, 2.9898252487182617]}, {"step": 5300, "epoch": 11, "train_loss": 14.389608383178711, "cos_r_negg": [0.08071089535951614, -0.40768879652023315, -0.13768839836120605], "cos_innovation_negg": [0.11131646484136581, 0.18680119514465332, 0.38463878631591797], "cos_apical_negg": [0.08071089535951614, -0.40768876671791077, -0.13768838346004486], "cos_Ac_negg": [0.11131632328033447, 0.186804860830307, 0.3846387565135956], "r_norm": [1.3056795597076416, 1.5338456630706787, 2.8537890911102295]}, {"step": 5400, "epoch": 11, "train_loss": 13.057826042175293, "cos_r_negg": [0.03638165444135666, -0.46403056383132935, -0.21357715129852295], "cos_innovation_negg": [-0.07257941365242004, -0.17930729687213898, 0.6900593042373657], "cos_apical_negg": [0.03638165816664696, -0.46403053402900696, -0.21357713639736176], "cos_Ac_negg": [-0.07257328927516937, -0.17930465936660767, 0.6900593042373657], "r_norm": [1.238492727279663, 1.4064722061157227, 2.8121910095214844]}, {"step": 5500, "epoch": 11, "train_loss": 12.050901412963867, "cos_r_negg": [0.008281117305159569, -0.16471955180168152, 0.12654852867126465], "cos_innovation_negg": [0.014256780967116356, 0.14428037405014038, 0.6110345125198364], "cos_apical_negg": [0.00828111544251442, -0.16471955180168152, 0.12654854357242584], "cos_Ac_negg": [0.014231878332793713, 0.144282266497612, 0.6110345125198364], "r_norm": [1.350577712059021, 1.5036232471466064, 2.9359333515167236], "ldr": {"dL_sdil": -1.9120216369628906, "dL_bp": -1.9120216369628906, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 14.115957260131836, "cos_r_negg": [0.0386815220117569, -0.42719346284866333, 0.09553170204162598], "cos_innovation_negg": [0.040902797132730484, 0.1142197996377945, 0.6166300773620605], "cos_apical_negg": [0.0386815220117569, -0.42719346284866333, 0.09553170204162598], "cos_Ac_negg": [0.040922343730926514, 0.11422167718410492, 0.6166300177574158], "r_norm": [1.2927098274230957, 1.4940705299377441, 3.021209716796875]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.0974, "test_loss": 11.0101029296875}, {"step": 5700, "epoch": 12, "train_loss": 15.534631729125977, "cos_r_negg": [0.022591736167669296, -0.5188312530517578, -0.06744025647640228], "cos_innovation_negg": [-0.1480846107006073, 0.14388906955718994, 0.7473440766334534], "cos_apical_negg": [0.022591741755604744, -0.5188312530517578, -0.06744025647640228], "cos_Ac_negg": [-0.1480652093887329, 0.1438835710287094, 0.7473440170288086], "r_norm": [1.3739748001098633, 1.5397405624389648, 3.237997055053711]}, {"step": 5800, "epoch": 12, "train_loss": 7.537947654724121, "cos_r_negg": [0.03714398667216301, -0.1610376536846161, 0.30687400698661804], "cos_innovation_negg": [-0.0189317986369133, 0.22660516202449799, 0.847710907459259], "cos_apical_negg": [0.03714399039745331, -0.1610376536846161, 0.30687403678894043], "cos_Ac_negg": [-0.018935054540634155, 0.22661373019218445, 0.847710907459259], "r_norm": [1.2435660362243652, 1.365397572517395, 2.8477783203125]}, {"step": 5900, "epoch": 12, "train_loss": 12.087260246276855, "cos_r_negg": [0.044624704867601395, -0.287490576505661, 0.14952147006988525], "cos_innovation_negg": [0.030776694416999817, 0.06920906901359558, 0.664570689201355], "cos_apical_negg": [0.04462470859289169, -0.287490576505661, 0.14952147006988525], "cos_Ac_negg": [0.03076918236911297, 0.06920908391475677, 0.664570689201355], "r_norm": [1.3061754703521729, 1.4420177936553955, 2.956468105316162]}, {"step": 6000, "epoch": 12, "train_loss": 9.213321685791016, "cos_r_negg": [0.032472848892211914, -0.44084417819976807, -0.007683856878429651], "cos_innovation_negg": [-0.02023286372423172, -0.028288723900914192, 0.7390176653862], "cos_apical_negg": [0.032472845166921616, -0.44084417819976807, -0.007683884352445602], "cos_Ac_negg": [-0.020207876339554787, -0.02828831784427166, 0.7390177845954895], "r_norm": [1.3800138235092163, 1.465118169784546, 3.2586922645568848], "ldr": {"dL_sdil": -3.6075868606567383, "dL_bp": -3.6075897216796875, "ratio": 0.9999992069433694}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.0892, "test_loss": 15.74776123046875}, {"step": 6100, "epoch": 13, "train_loss": 13.45030689239502, "cos_r_negg": [-0.026875589042901993, -0.22675372660160065, 0.07094521820545197], "cos_innovation_negg": [0.01712423376739025, -0.07511216402053833, 0.7534027099609375], "cos_apical_negg": [-0.02687559276819229, -0.22675372660160065, 0.07094522565603256], "cos_Ac_negg": [0.017143916338682175, -0.07511210441589355, 0.7534027099609375], "r_norm": [1.1944451332092285, 1.3047837018966675, 2.8725621700286865]}, {"step": 6200, "epoch": 13, "train_loss": 12.124948501586914, "cos_r_negg": [0.0250451248139143, -0.4816763401031494, -0.1524362862110138], "cos_innovation_negg": [-0.05051799118518829, 0.43737471103668213, 0.7273811101913452], "cos_apical_negg": [0.025045115500688553, -0.4816763401031494, -0.1524362862110138], "cos_Ac_negg": [-0.0525033101439476, 0.4373703896999359, 0.7273818850517273], "r_norm": [1.4082647562026978, 1.6057106256484985, 3.4653730392456055]}, {"step": 6300, "epoch": 13, "train_loss": 8.079349517822266, "cos_r_negg": [0.07201477885246277, -0.40998172760009766, -0.3364822268486023], "cos_innovation_negg": [-0.03286297246813774, -0.232041135430336, 0.3800865709781647], "cos_apical_negg": [0.07201477885246277, -0.40998172760009766, -0.3364821672439575], "cos_Ac_negg": [-0.03286667540669441, -0.2320411503314972, 0.38008660078048706], "r_norm": [1.21011221408844, 1.3437243700027466, 3.0055012702941895]}, {"step": 6400, "epoch": 13, "train_loss": 7.763940811157227, "cos_r_negg": [0.05184975266456604, -0.37042123079299927, 0.10609821230173111], "cos_innovation_negg": [0.12897612154483795, 0.22582663595676422, 0.9743067622184753], "cos_apical_negg": [0.05184975266456604, -0.37042123079299927, 0.1060982197523117], "cos_Ac_negg": [0.12866821885108948, 0.2258259654045105, 0.9743068218231201], "r_norm": [1.2864501476287842, 1.26600182056427, 3.4056525230407715]}, {"step": 6500, "epoch": 13, "train_loss": 9.023736953735352, "cos_r_negg": [0.029297159984707832, -0.36023253202438354, 0.11098258942365646], "cos_innovation_negg": [-0.007821168750524521, 0.2545849084854126, 0.9241091012954712], "cos_apical_negg": [0.029297158122062683, -0.36023253202438354, 0.11098258197307587], "cos_Ac_negg": [-0.007846848107874393, 0.2545849084854126, 0.9241091012954712], "r_norm": [1.1800556182861328, 1.3715802431106567, 3.0239391326904297], "ldr": {"dL_sdil": -1.5745134353637695, "dL_bp": -1.5745134353637695, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.098, "test_loss": 14.1557130859375}, {"step": 6600, "epoch": 14, "train_loss": 11.782710075378418, "cos_r_negg": [0.08973315358161926, -0.38965848088264465, -0.15500372648239136], "cos_innovation_negg": [0.12626081705093384, -0.14959390461444855, 0.8248876333236694], "cos_apical_negg": [0.08973314613103867, -0.38965845108032227, -0.15500372648239136], "cos_Ac_negg": [0.126374751329422, -0.14959387481212616, 0.8248878121376038], "r_norm": [1.255458950996399, 1.3828380107879639, 3.345825672149658]}, {"step": 6700, "epoch": 14, "train_loss": 9.433488845825195, "cos_r_negg": [-0.004701629281044006, -0.28984975814819336, -0.24890448153018951], "cos_innovation_negg": [-0.11351288855075836, -0.34793803095817566, 0.33490055799484253], "cos_apical_negg": [-0.004701625555753708, -0.28984975814819336, -0.24890448153018951], "cos_Ac_negg": [-0.1135249137878418, -0.3479382395744324, 0.334900438785553], "r_norm": [1.3931225538253784, 1.794011116027832, 3.601773738861084]}, {"step": 6800, "epoch": 14, "train_loss": 10.120220184326172, "cos_r_negg": [0.009237721562385559, -0.28421688079833984, -0.05856458470225334], "cos_innovation_negg": [0.03159017115831375, 0.19670310616493225, 0.9492440223693848], "cos_apical_negg": [0.00923771969974041, -0.28421688079833984, -0.058564599603414536], "cos_Ac_negg": [0.03157833218574524, 0.19670310616493225, 0.9492440223693848], "r_norm": [1.1015956401824951, 1.153090476989746, 2.935507297515869]}, {"step": 6900, "epoch": 14, "train_loss": 6.735630512237549, "cos_r_negg": [0.023455128073692322, -0.3958374857902527, -0.15575039386749268], "cos_innovation_negg": [0.006992906332015991, 0.08461026847362518, 0.95595383644104], "cos_apical_negg": [0.023455128073692322, -0.3958375155925751, -0.15575037896633148], "cos_Ac_negg": [0.006896665319800377, 0.0846102386713028, 0.95595383644104], "r_norm": [1.0282623767852783, 1.1017005443572998, 2.8485217094421387]}, {"step": 7000, "epoch": 14, "train_loss": 14.088212013244629, "cos_r_negg": [0.05030996724963188, 0.019453780725598335, 0.07658995687961578], "cos_innovation_negg": [0.055587038397789, 0.04014395549893379, 0.8051148653030396], "cos_apical_negg": [0.05030997470021248, 0.019453778862953186, 0.07658997178077698], "cos_Ac_negg": [0.0555681511759758, 0.0401439405977726, 0.8051148653030396], "r_norm": [1.1992748975753784, 1.3771302700042725, 3.2397217750549316], "ldr": {"dL_sdil": -2.3611583709716797, "dL_bp": -2.3611583709716797, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1135, "test_loss": 10.0383939453125}], "final": {"test_acc": 0.1135, "test_loss": 10.0383939453125, "wall_s": 25.687421321868896, "cos_r_negg": [0.023699067533016205, -0.41677844524383545, 0.002772160340100527], "cos_innovation_negg": [0.056335173547267914, 0.1732165515422821, 0.9118537902832031], "cos_apical_negg": [0.023699071258306503, -0.41677841544151306, 0.0027721733786165714], "cos_Ac_negg": [0.056318968534469604, 0.1732165664434433, 0.9118537902832031]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_matched_s3.json b/results/nuis_ctrl_v1_rho0p5_matched_s3.json new file mode 100644 index 0000000..61f8f23 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_matched_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_matched_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.023404492065310478, 0.009263666346669197, -0.003423936665058136], "cos_innovation_negg": [-0.023943275213241577, 0.015888750553131104, 0.006476129405200481], "cos_apical_negg": [-0.023404492065310478, 0.009263665415346622, -0.0034239361993968487], "cos_Ac_negg": [-0.023945096880197525, 0.015887551009655, 0.006474699825048447], "r_norm": [4.880455017089844, 4.766017913818359, 4.871740341186523], "ldr": {"dL_sdil": 0.05150461196899414, "dL_bp": -0.2127234935760498, "ratio": -0.24211999861021916}}, {"step": 100, "epoch": 0, "train_loss": 3.93803071975708, "cos_r_negg": [-0.05274544283747673, 0.18825885653495789, -0.44922441244125366], "cos_innovation_negg": [-0.09187984466552734, -0.00649038702249527, -0.33347389101982117], "cos_apical_negg": [-0.05274543911218643, 0.18825885653495789, -0.44922447204589844], "cos_Ac_negg": [-0.09188588708639145, -0.006491729989647865, -0.33348870277404785], "r_norm": [5.036410331726074, 4.923680305480957, 5.091097831726074]}, {"step": 200, "epoch": 0, "train_loss": 12.228022575378418, "cos_r_negg": [-0.017211202532052994, 0.05333784595131874, 0.36261531710624695], "cos_innovation_negg": [-0.04136255010962486, 0.09066534042358398, 0.14909055829048157], "cos_apical_negg": [-0.017211204394698143, 0.053337857127189636, 0.36261534690856934], "cos_Ac_negg": [-0.04134999215602875, 0.09060462564229965, 0.1490904688835144], "r_norm": [5.575984001159668, 5.387224197387695, 5.670280933380127]}, {"step": 300, "epoch": 0, "train_loss": 9.527034759521484, "cos_r_negg": [0.09905003011226654, 0.11598698794841766, -0.337249219417572], "cos_innovation_negg": [0.13977257907390594, 0.2395804226398468, -0.057061560451984406], "cos_apical_negg": [0.09905002266168594, 0.11598698794841766, -0.33724915981292725], "cos_Ac_negg": [0.13977400958538055, 0.2395714521408081, -0.05707992985844612], "r_norm": [5.27436637878418, 5.377788543701172, 5.334036350250244]}, {"step": 400, "epoch": 0, "train_loss": 10.18307876586914, "cos_r_negg": [-0.016085131093859673, -0.22224563360214233, -0.10172560811042786], "cos_innovation_negg": [-0.06050053983926773, -0.0628795176744461, 0.5877997875213623], "cos_apical_negg": [-0.016085131093859673, -0.22224563360214233, -0.10172560811042786], "cos_Ac_negg": [-0.06043291836977005, -0.06283138692378998, 0.5877983570098877], "r_norm": [5.364789009094238, 5.296049118041992, 5.427796840667725]}, {"epoch_end": 0, "step": 469, "test_acc": 0.098, "test_loss": 13.6578900390625}, {"step": 500, "epoch": 1, "train_loss": 11.668904304504395, "cos_r_negg": [-0.023274647071957588, 0.11952609568834305, -0.2331356406211853], "cos_innovation_negg": [-0.016597865149378777, 0.12962821125984192, -0.21243219077587128], "cos_apical_negg": [-0.023274652659893036, 0.11952609568834305, -0.23313559591770172], "cos_Ac_negg": [-0.016598738729953766, 0.12961795926094055, -0.2124209702014923], "r_norm": [4.465723991394043, 4.6360955238342285, 4.811782360076904], "ldr": {"dL_sdil": -1.660008430480957, "dL_bp": -1.660008430480957, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 9.913995742797852, "cos_r_negg": [0.03544580936431885, 0.010761044919490814, 0.11519358307123184], "cos_innovation_negg": [0.024819735437631607, 0.13745856285095215, 0.30716899037361145], "cos_apical_negg": [0.03544580191373825, 0.010761039331555367, 0.11519362777471542], "cos_Ac_negg": [0.024800240993499756, 0.13745294511318207, 0.3071695566177368], "r_norm": [4.528250694274902, 4.526756286621094, 4.732523441314697]}, {"step": 700, "epoch": 1, "train_loss": 11.596376419067383, "cos_r_negg": [-0.016515249386429787, -0.024830589070916176, -0.6743102073669434], "cos_innovation_negg": [-0.0237373486161232, 0.10925282537937164, -0.04644819349050522], "cos_apical_negg": [-0.016515253111720085, -0.024830587208271027, -0.6743102669715881], "cos_Ac_negg": [-0.024258635938167572, 0.10920343548059464, -0.04642816632986069], "r_norm": [4.360329627990723, 4.562563896179199, 4.7867584228515625]}, {"step": 800, "epoch": 1, "train_loss": 16.685710906982422, "cos_r_negg": [0.05001528561115265, -0.15051516890525818, -0.476337194442749], "cos_innovation_negg": [0.03804410994052887, 0.06662282347679138, 0.6450439691543579], "cos_apical_negg": [0.05001528188586235, -0.15051516890525818, -0.47633713483810425], "cos_Ac_negg": [0.03804052621126175, 0.06666028499603271, 0.6450420618057251], "r_norm": [4.281778335571289, 4.418721675872803, 4.574766159057617]}, {"step": 900, "epoch": 1, "train_loss": 8.289448738098145, "cos_r_negg": [-0.03426763042807579, -0.14169526100158691, -0.39391225576400757], "cos_innovation_negg": [-0.10632513463497162, 0.06073577329516411, -0.07448279857635498], "cos_apical_negg": [-0.03426763042807579, -0.14169526100158691, -0.3939122259616852], "cos_Ac_negg": [-0.10630496591329575, 0.0607910230755806, -0.07447711378335953], "r_norm": [3.7901594638824463, 4.016811847686768, 4.248052597045898]}, {"epoch_end": 1, "step": 938, "test_acc": 0.1028, "test_loss": 9.41012841796875}, {"step": 1000, "epoch": 2, "train_loss": 12.244726181030273, "cos_r_negg": [-0.004659376572817564, -0.005834559444338083, -0.5812898874282837], "cos_innovation_negg": [-0.014617500826716423, 0.18938259780406952, -0.0436372235417366], "cos_apical_negg": [-0.0046593728475272655, -0.005834552459418774, -0.5812898874282837], "cos_Ac_negg": [-0.014623701572418213, 0.18937918543815613, -0.04364272952079773], "r_norm": [3.4024252891540527, 3.7132411003112793, 3.841618061065674], "ldr": {"dL_sdil": -4.327396392822266, "dL_bp": -4.327396392822266, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 13.012718200683594, "cos_r_negg": [0.035421840846538544, 0.07751062512397766, 0.29841554164886475], "cos_innovation_negg": [0.034294068813323975, 0.33310723304748535, 0.41063791513442993], "cos_apical_negg": [0.03542184457182884, 0.07751062512397766, 0.29841557145118713], "cos_Ac_negg": [0.033763423562049866, 0.3331449031829834, 0.4106253385543823], "r_norm": [3.697913646697998, 3.8519930839538574, 4.062531471252441]}, {"step": 1200, "epoch": 2, "train_loss": 13.053409576416016, "cos_r_negg": [0.004354442469775677, -0.117958202958107, -0.7837634086608887], "cos_innovation_negg": [-0.0614975206553936, 0.0449785478413105, -0.6589546799659729], "cos_apical_negg": [0.004354442469775677, -0.117958202958107, -0.7837634086608887], "cos_Ac_negg": [-0.05989518016576767, 0.023895755410194397, -0.6684364080429077], "r_norm": [3.6265907287597656, 4.030216217041016, 4.190539360046387]}, {"step": 1300, "epoch": 2, "train_loss": 5.807992458343506, "cos_r_negg": [0.021018171682953835, 0.05074962228536606, 0.2531043589115143], "cos_innovation_negg": [0.01744532585144043, 0.04455544427037239, 0.11088324338197708], "cos_apical_negg": [0.021018177270889282, 0.050749629735946655, 0.2531043291091919], "cos_Ac_negg": [0.017444536089897156, 0.044559065252542496, 0.11088646948337555], "r_norm": [2.9064719676971436, 3.20048451423645, 3.3757195472717285]}, {"step": 1400, "epoch": 2, "train_loss": 16.68470001220703, "cos_r_negg": [-0.02570568025112152, -0.21907366812229156, -0.7805821299552917], "cos_innovation_negg": [-0.0555734746158123, 0.1145012378692627, -0.4349041283130646], "cos_apical_negg": [-0.025705672800540924, -0.21907366812229156, -0.7805821299552917], "cos_Ac_negg": [-0.055576201528310776, 0.11466749012470245, -0.4349042475223541], "r_norm": [3.105128765106201, 3.472952365875244, 3.6450729370117188]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.0974, "test_loss": 16.10644501953125}, {"step": 1500, "epoch": 3, "train_loss": 10.99614429473877, "cos_r_negg": [-0.010981716215610504, -0.10529394447803497, -0.6657657623291016], "cos_innovation_negg": [-0.04375379532575607, -0.006582379341125488, 0.470498651266098], "cos_apical_negg": [-0.010981718078255653, -0.10529392957687378, -0.6657657623291016], "cos_Ac_negg": [-0.043721724301576614, -0.006521882489323616, 0.4704965651035309], "r_norm": [3.093276023864746, 3.3588638305664062, 3.678278923034668], "ldr": {"dL_sdil": -3.1172027587890625, "dL_bp": -3.1172027587890625, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 13.572635650634766, "cos_r_negg": [0.032471850514411926, 0.09270533919334412, -0.5350372195243835], "cos_innovation_negg": [0.08556779474020004, 0.13869449496269226, 0.20785954594612122], "cos_apical_negg": [0.032471850514411926, 0.09270535409450531, -0.5350371599197388], "cos_Ac_negg": [0.08557653427124023, 0.13869059085845947, 0.20785681903362274], "r_norm": [2.8790063858032227, 3.2431540489196777, 3.545623779296875]}, {"step": 1700, "epoch": 3, "train_loss": 15.694596290588379, "cos_r_negg": [0.033936936408281326, -0.08957314491271973, -0.6843352317810059], "cos_innovation_negg": [-0.012293849140405655, 0.010994723066687584, 0.40581268072128296], "cos_apical_negg": [0.033936940133571625, -0.08957314491271973, -0.6843352913856506], "cos_Ac_negg": [-0.012302355840802193, 0.011027035303413868, 0.4058120548725128], "r_norm": [2.7849206924438477, 3.0635476112365723, 3.429154396057129]}, {"step": 1800, "epoch": 3, "train_loss": 16.105371475219727, "cos_r_negg": [0.021749984472990036, -0.14429184794425964, -0.6999454498291016], "cos_innovation_negg": [0.017163287848234177, 0.11140023171901703, 0.14725594222545624], "cos_apical_negg": [0.021749980747699738, -0.14429183304309845, -0.6999454498291016], "cos_Ac_negg": [0.01746225915849209, 0.11156095564365387, 0.14757612347602844], "r_norm": [2.761136531829834, 3.095857858657837, 3.5029616355895996]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.0982, "test_loss": 11.78546201171875}, {"step": 1900, "epoch": 4, "train_loss": 8.233247756958008, "cos_r_negg": [0.0118138138204813, -0.052501268684864044, 0.2882298231124878], "cos_innovation_negg": [-0.014771915040910244, 0.2622224986553192, 0.34787923097610474], "cos_apical_negg": [0.011813816614449024, -0.052501268684864044, 0.2882297933101654], "cos_Ac_negg": [-0.014770178124308586, 0.26222601532936096, 0.34787771105766296], "r_norm": [2.276346206665039, 2.6683669090270996, 2.9762845039367676]}, {"step": 2000, "epoch": 4, "train_loss": 10.540949821472168, "cos_r_negg": [-0.002448295708745718, 0.046116843819618225, -0.36156171560287476], "cos_innovation_negg": [0.020891379565000534, 0.04625238850712776, 0.1458940953016281], "cos_apical_negg": [-0.002448292449116707, 0.04611684009432793, -0.36156171560287476], "cos_Ac_negg": [0.02089184708893299, 0.04623724892735481, 0.1458939015865326], "r_norm": [2.3217287063598633, 2.6515660285949707, 3.0398008823394775], "ldr": {"dL_sdil": -1.352426528930664, "dL_bp": -1.352426528930664, "ratio": 1.0}}, {"step": 2100, "epoch": 4, "train_loss": 9.533159255981445, "cos_r_negg": [0.0063061583787202835, 0.07012288272380829, -0.20739030838012695], "cos_innovation_negg": [0.03759526461362839, 0.38612493872642517, 0.445768803358078], "cos_apical_negg": [0.0063061523251235485, 0.07012288272380829, -0.20739032328128815], "cos_Ac_negg": [0.037591226398944855, 0.3861226737499237, 0.4457683563232422], "r_norm": [2.1785190105438232, 2.5417747497558594, 2.8388586044311523]}, {"step": 2200, "epoch": 4, "train_loss": 7.513348579406738, "cos_r_negg": [-0.0007374449633061886, -0.00730521697551012, 0.04074469208717346], "cos_innovation_negg": [0.017737476155161858, 0.3992329239845276, 0.36693888902664185], "cos_apical_negg": [-0.0007374389097094536, -0.007305213250219822, 0.04074469208717346], "cos_Ac_negg": [0.017734572291374207, 0.3992356061935425, 0.36693844199180603], "r_norm": [2.142125368118286, 2.489021062850952, 2.871180534362793]}, {"step": 2300, "epoch": 4, "train_loss": 13.4251708984375, "cos_r_negg": [0.03304494544863701, 0.03516755998134613, -0.23744675517082214], "cos_innovation_negg": [0.0314888060092926, 0.23710165917873383, 0.3066062927246094], "cos_apical_negg": [0.03304494917392731, 0.03516756370663643, -0.23744671046733856], "cos_Ac_negg": [0.03148419409990311, 0.2371007353067398, 0.3066060245037079], "r_norm": [2.0966086387634277, 2.517566680908203, 2.866859197616577]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.1009, "test_loss": 9.70174755859375}, {"step": 2400, "epoch": 5, "train_loss": 12.111278533935547, "cos_r_negg": [0.04637152701616287, 0.03708937391638756, -0.44551995396614075], "cos_innovation_negg": [-0.011598706245422363, 0.29721200466156006, 0.18614904582500458], "cos_apical_negg": [0.046371523290872574, 0.03708937391638756, -0.44551992416381836], "cos_Ac_negg": [-0.01160514261573553, 0.2972118854522705, 0.18614903092384338], "r_norm": [1.9773354530334473, 2.295269012451172, 2.829575538635254]}, {"step": 2500, "epoch": 5, "train_loss": 12.332671165466309, "cos_r_negg": [0.041929714381694794, -0.06426175683736801, 0.5717096328735352], "cos_innovation_negg": [0.07432641088962555, 0.18175101280212402, 0.6384155750274658], "cos_apical_negg": [0.04192971810698509, -0.06426175683736801, 0.5717096328735352], "cos_Ac_negg": [0.0743139386177063, 0.18174266815185547, 0.6384155750274658], "r_norm": [2.136064291000366, 2.6100192070007324, 3.3647665977478027], "ldr": {"dL_sdil": -2.3579254150390625, "dL_bp": -2.3579254150390625, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 14.47452163696289, "cos_r_negg": [-0.013248052448034286, 0.05644286423921585, -0.7643646001815796], "cos_innovation_negg": [-0.17318934202194214, 0.050919756293296814, -0.022107506170868874], "cos_apical_negg": [-0.013248054310679436, 0.05644286423921585, -0.7643646001815796], "cos_Ac_negg": [-0.1731943041086197, 0.050827860832214355, -0.02210751362144947], "r_norm": [2.2183806896209717, 2.5999670028686523, 3.0638198852539062]}, {"step": 2700, "epoch": 5, "train_loss": 7.529302597045898, "cos_r_negg": [0.06156715005636215, 0.09324220567941666, 0.5201743841171265], "cos_innovation_negg": [0.05897732824087143, 0.3124152421951294, -0.12993979454040527], "cos_apical_negg": [0.06156715005636215, 0.09324220567941666, 0.5201743841171265], "cos_Ac_negg": [0.05895711109042168, 0.3116137385368347, -0.1299402415752411], "r_norm": [2.144774913787842, 2.3828392028808594, 2.95363712310791]}, {"step": 2800, "epoch": 5, "train_loss": 11.245426177978516, "cos_r_negg": [0.021822623908519745, -0.07685096561908722, 0.1306837499141693], "cos_innovation_negg": [0.052651163190603256, 0.024222731590270996, 0.4682297110557556], "cos_apical_negg": [0.021822625771164894, -0.07685096561908722, 0.1306837499141693], "cos_Ac_negg": [0.05265333503484726, 0.024229511618614197, 0.46822965145111084], "r_norm": [1.837006688117981, 2.098672389984131, 2.7745492458343506]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.0982, "test_loss": 9.00151484375}, {"step": 2900, "epoch": 6, "train_loss": 10.699666023254395, "cos_r_negg": [0.003275001421570778, 0.01193692535161972, 0.5769363641738892], "cos_innovation_negg": [0.0802985355257988, 0.21270713210105896, 0.886192262172699], "cos_apical_negg": [0.003275001421570778, 0.011936930939555168, 0.5769364237785339], "cos_Ac_negg": [0.08028008043766022, 0.2127087414264679, 0.8861921429634094], "r_norm": [1.9166699647903442, 2.342777729034424, 3.2197136878967285]}, {"step": 3000, "epoch": 6, "train_loss": 8.966672897338867, "cos_r_negg": [0.012364748865365982, -0.06855656951665878, -0.5414963364601135], "cos_innovation_negg": [0.04855531081557274, 0.25342071056365967, 0.15319502353668213], "cos_apical_negg": [0.012364747002720833, -0.06855656951665878, -0.5414963364601135], "cos_Ac_negg": [0.04855132848024368, 0.25339818000793457, 0.15319499373435974], "r_norm": [1.7323641777038574, 1.9787015914916992, 2.7299466133117676], "ldr": {"dL_sdil": -1.3645620346069336, "dL_bp": -1.3645620346069336, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 7.404589653015137, "cos_r_negg": [-0.00993923656642437, 0.03794018179178238, -0.2275383174419403], "cos_innovation_negg": [0.05834876000881195, 0.31114208698272705, 0.4983579218387604], "cos_apical_negg": [-0.009939240291714668, 0.03794018179178238, -0.2275383174419403], "cos_Ac_negg": [0.05834975466132164, 0.31113043427467346, 0.498357892036438], "r_norm": [1.839612603187561, 2.121492862701416, 2.895768404006958]}, {"step": 3200, "epoch": 6, "train_loss": 13.985771179199219, "cos_r_negg": [0.009052676148712635, 0.044186364859342575, -0.6484626531600952], "cos_innovation_negg": [0.007429224438965321, -0.08652340620756149, 0.35605454444885254], "cos_apical_negg": [0.00905267521739006, 0.044186364859342575, -0.6484626531600952], "cos_Ac_negg": [0.007442607544362545, -0.08654331415891647, 0.35605478286743164], "r_norm": [1.8577483892440796, 2.176766872406006, 3.1609139442443848]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.1032, "test_loss": 9.5078845703125}, {"step": 3300, "epoch": 7, "train_loss": 14.500346183776855, "cos_r_negg": [0.03023715317249298, 0.05987554416060448, 0.10631096363067627], "cos_innovation_negg": [0.01239866204559803, 0.21723376214504242, 0.3753962516784668], "cos_apical_negg": [0.03023715876042843, 0.05987554043531418, 0.10631096363067627], "cos_Ac_negg": [0.010563991032540798, 0.21948036551475525, 0.37538307905197144], "r_norm": [1.9232898950576782, 2.158068895339966, 2.9749834537506104]}, {"step": 3400, "epoch": 7, "train_loss": 8.561042785644531, "cos_r_negg": [-0.007763438392430544, 0.03267252817749977, -0.5633492469787598], "cos_innovation_negg": [0.005812717601656914, 0.08363096415996552, 0.49122267961502075], "cos_apical_negg": [-0.007763439789414406, 0.032672520726919174, -0.5633492469787598], "cos_Ac_negg": [0.005812252406030893, 0.0836193859577179, 0.491222620010376], "r_norm": [1.8085782527923584, 2.004695415496826, 2.943936824798584]}, {"step": 3500, "epoch": 7, "train_loss": 6.397796154022217, "cos_r_negg": [-0.009914370253682137, 0.10034030675888062, -0.46877723932266235], "cos_innovation_negg": [-0.003673017956316471, 0.16520525515079498, 0.35941267013549805], "cos_apical_negg": [-0.009914370253682137, 0.10034029930830002, -0.46877723932266235], "cos_Ac_negg": [-0.0036725131794810295, 0.16517749428749084, 0.35941267013549805], "r_norm": [1.6035305261611938, 1.8609508275985718, 2.7025599479675293], "ldr": {"dL_sdil": -1.8581633567810059, "dL_bp": -1.8581633567810059, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 8.999161720275879, "cos_r_negg": [-0.02817051112651825, 0.03443991392850876, -0.6405396461486816], "cos_innovation_negg": [-0.10997366905212402, -0.2562880516052246, 0.07866506278514862], "cos_apical_negg": [-0.0281705092638731, 0.03443991392850876, -0.6405396461486816], "cos_Ac_negg": [-0.10997616499662399, -0.2562910318374634, 0.07866504788398743], "r_norm": [1.6904629468917847, 1.9610121250152588, 2.7385761737823486]}, {"step": 3700, "epoch": 7, "train_loss": 14.690881729125977, "cos_r_negg": [-0.008290690369904041, 0.07577173411846161, -0.6582216620445251], "cos_innovation_negg": [-0.06754305958747864, -0.10545620322227478, 0.6658633947372437], "cos_apical_negg": [-0.008290693163871765, 0.0757717415690422, -0.6582217216491699], "cos_Ac_negg": [-0.06750993430614471, -0.10544462502002716, 0.6658633947372437], "r_norm": [1.7294771671295166, 2.0196428298950195, 3.1941299438476562]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.1009, "test_loss": 12.96227607421875}, {"step": 3800, "epoch": 8, "train_loss": 10.057510375976562, "cos_r_negg": [-0.04070144519209862, 0.08962370455265045, -0.7099792957305908], "cos_innovation_negg": [-0.1700393259525299, -0.39376434683799744, 0.015467292629182339], "cos_apical_negg": [-0.04070143774151802, 0.08962368965148926, -0.7099792957305908], "cos_Ac_negg": [-0.16983087360858917, -0.3937705159187317, 0.015468060038983822], "r_norm": [1.814821720123291, 2.1310102939605713, 2.9847631454467773]}, {"step": 3900, "epoch": 8, "train_loss": 8.215349197387695, "cos_r_negg": [-0.0048723118379712105, 0.09534664452075958, -0.4127228260040283], "cos_innovation_negg": [-0.03572871536016464, -0.4212097227573395, 0.7570983171463013], "cos_apical_negg": [-0.004872307181358337, 0.09534664452075958, -0.4127228260040283], "cos_Ac_negg": [-0.035726048052310944, -0.42121100425720215, 0.7570985555648804], "r_norm": [1.6550533771514893, 1.9254438877105713, 3.1527276039123535]}, {"step": 4000, "epoch": 8, "train_loss": 10.831953048706055, "cos_r_negg": [-0.020632298663258553, 0.050789255648851395, -0.15508905053138733], "cos_innovation_negg": [-0.07258114218711853, -0.5072916150093079, 0.5620717406272888], "cos_apical_negg": [-0.020632293075323105, 0.050789255648851395, -0.15508908033370972], "cos_Ac_negg": [-0.07259061932563782, -0.5072973966598511, 0.5620717406272888], "r_norm": [1.445892095565796, 1.6738638877868652, 2.6616740226745605], "ldr": {"dL_sdil": -2.0770139694213867, "dL_bp": -2.0770139694213867, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 13.534512519836426, "cos_r_negg": [0.03284500539302826, 0.1665118932723999, -0.3395427167415619], "cos_innovation_negg": [-0.11990010738372803, -0.18010328710079193, 0.40169036388397217], "cos_apical_negg": [0.03284499794244766, 0.1665118932723999, -0.3395427167415619], "cos_Ac_negg": [-0.11991329491138458, -0.1801161766052246, 0.40169036388397217], "r_norm": [1.5067338943481445, 1.6453357934951782, 2.785231590270996]}, {"step": 4200, "epoch": 8, "train_loss": 10.042614936828613, "cos_r_negg": [-0.009762363508343697, -0.010503213852643967, -0.6587464213371277], "cos_innovation_negg": [-0.1768125295639038, -0.3322966694831848, 0.32998669147491455], "cos_apical_negg": [-0.009762364439666271, -0.010503217577934265, -0.6587464809417725], "cos_Ac_negg": [-0.1767541468143463, -0.33232879638671875, 0.32998669147491455], "r_norm": [1.6771917343139648, 1.9635087251663208, 2.9597654342651367]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.0958, "test_loss": 6.23742001953125}, {"step": 4300, "epoch": 9, "train_loss": 10.101000785827637, "cos_r_negg": [0.008505766279995441, -0.045377250760793686, -0.6276191473007202], "cos_innovation_negg": [-0.12647788226604462, -0.24161115288734436, 0.3345617949962616], "cos_apical_negg": [0.008505766279995441, -0.045377250760793686, -0.6276191473007202], "cos_Ac_negg": [-0.1264556646347046, -0.24159809947013855, 0.3345620632171631], "r_norm": [1.6475998163223267, 1.9239721298217773, 2.967597484588623]}, {"step": 4400, "epoch": 9, "train_loss": 9.596513748168945, "cos_r_negg": [0.008562065660953522, 0.15001876652240753, -0.175533264875412], "cos_innovation_negg": [-0.04514005407691002, -0.48557454347610474, 0.6526634693145752], "cos_apical_negg": [0.008562060073018074, 0.15001878142356873, -0.17553327977657318], "cos_Ac_negg": [-0.045139163732528687, -0.48565950989723206, 0.6526634693145752], "r_norm": [1.4307036399841309, 1.6210697889328003, 2.8097445964813232]}, {"step": 4500, "epoch": 9, "train_loss": 16.366668701171875, "cos_r_negg": [-0.024215523153543472, 0.0523746982216835, -0.13722029328346252], "cos_innovation_negg": [-0.06498889625072479, -0.1888183355331421, 0.29898589849472046], "cos_apical_negg": [-0.02421552501618862, 0.0523746982216835, -0.13722029328346252], "cos_Ac_negg": [-0.06498885899782181, -0.18893036246299744, 0.29898592829704285], "r_norm": [1.4138031005859375, 1.6285908222198486, 2.8013172149658203], "ldr": {"dL_sdil": -3.148632049560547, "dL_bp": -3.148632049560547, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 15.175790786743164, "cos_r_negg": [-0.001095588319003582, 0.017703622579574585, -0.6027648448944092], "cos_innovation_negg": [0.01033240370452404, -0.28939884901046753, 0.6939722299575806], "cos_apical_negg": [-0.0010955953039228916, 0.017703616991639137, -0.6027647852897644], "cos_Ac_negg": [0.01495253574103117, -0.37095627188682556, 0.6939723491668701], "r_norm": [1.5095683336257935, 1.7129693031311035, 3.209284543991089]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.0982, "test_loss": 10.176494140625}, {"step": 4700, "epoch": 10, "train_loss": 12.548572540283203, "cos_r_negg": [0.03662903606891632, 0.013323778286576271, 0.03179753199219704], "cos_innovation_negg": [-0.11625862121582031, -0.22014150023460388, 0.5986127853393555], "cos_apical_negg": [0.03662903606891632, 0.013323778286576271, 0.03179751709103584], "cos_Ac_negg": [-0.11626827716827393, -0.22014975547790527, 0.5986127853393555], "r_norm": [1.3283519744873047, 1.469618320465088, 2.7055137157440186]}, {"step": 4800, "epoch": 10, "train_loss": 10.963335037231445, "cos_r_negg": [-0.021155945956707, 0.3539727032184601, 0.5038083791732788], "cos_innovation_negg": [-0.15960907936096191, 0.0907750278711319, 0.729169487953186], "cos_apical_negg": [-0.02115594409406185, 0.3539727032184601, 0.5038083791732788], "cos_Ac_negg": [-0.1595846712589264, 0.09067634493112564, 0.729169487953186], "r_norm": [1.4317306280136108, 1.7174842357635498, 3.339275360107422]}, {"step": 4900, "epoch": 10, "train_loss": 10.232675552368164, "cos_r_negg": [0.07411853969097137, -0.1702471375465393, -0.48597821593284607], "cos_innovation_negg": [0.06541065871715546, 0.2892952859401703, 0.9796814322471619], "cos_apical_negg": [0.07411853969097137, -0.1702471524477005, -0.4859781861305237], "cos_Ac_negg": [0.06539630889892578, 0.2896279990673065, 0.9796813726425171], "r_norm": [1.5119335651397705, 1.5873453617095947, 3.180260419845581]}, {"step": 5000, "epoch": 10, "train_loss": 13.096416473388672, "cos_r_negg": [0.03610732778906822, 0.15713843703269958, -0.291078120470047], "cos_innovation_negg": [0.0072157131507992744, -0.3843618631362915, 0.8875001072883606], "cos_apical_negg": [0.036107324063777924, 0.15713843703269958, -0.291078120470047], "cos_Ac_negg": [0.007210638374090195, -0.38435596227645874, 0.8875000476837158], "r_norm": [1.2840259075164795, 1.4135812520980835, 2.7617764472961426], "ldr": {"dL_sdil": -2.350564956665039, "dL_bp": -2.350564956665039, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 10.827922821044922, "cos_r_negg": [0.024950947612524033, 0.2864372730255127, 0.02290518581867218], "cos_innovation_negg": [-0.08479246497154236, -0.31673842668533325, 0.4500734210014343], "cos_apical_negg": [0.024950947612524033, 0.2864372134208679, 0.022905167192220688], "cos_Ac_negg": [-0.08480361104011536, -0.31676894426345825, 0.4500734210014343], "r_norm": [1.2995893955230713, 1.452526569366455, 2.8131027221679688]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.098, "test_loss": 14.10288837890625}, {"step": 5200, "epoch": 11, "train_loss": 9.503578186035156, "cos_r_negg": [0.013169646263122559, 0.08294951915740967, -0.6244248151779175], "cos_innovation_negg": [-0.11085792630910873, -0.37744754552841187, 0.45521920919418335], "cos_apical_negg": [0.01316964440047741, 0.08294951915740967, -0.6244248151779175], "cos_Ac_negg": [-0.11086191236972809, -0.37721192836761475, 0.4552191197872162], "r_norm": [1.2747329473495483, 1.3942285776138306, 2.767704963684082]}, {"step": 5300, "epoch": 11, "train_loss": 5.430373191833496, "cos_r_negg": [0.003571514505892992, -0.16799697279930115, -0.4963197708129883], "cos_innovation_negg": [0.0434497594833374, 0.3517071604728699, 0.8774371147155762], "cos_apical_negg": [0.003571515902876854, -0.16799697279930115, -0.4963197708129883], "cos_Ac_negg": [0.04343333840370178, 0.35168662667274475, 0.8774371147155762], "r_norm": [1.3673832416534424, 1.4287078380584717, 3.0254065990448]}, {"step": 5400, "epoch": 11, "train_loss": 14.83660888671875, "cos_r_negg": [0.005606683902442455, -0.10506697744131088, -0.5611649751663208], "cos_innovation_negg": [-0.0940738320350647, -0.2691645324230194, 0.7712353467941284], "cos_apical_negg": [0.005606682039797306, -0.10506697744131088, -0.5611649751663208], "cos_Ac_negg": [-0.09403727948665619, -0.2690330445766449, 0.7712351679801941], "r_norm": [1.351412057876587, 1.4972279071807861, 3.23873233795166]}, {"step": 5500, "epoch": 11, "train_loss": 11.84660530090332, "cos_r_negg": [0.04608001559972763, 0.0542776994407177, -0.14153742790222168], "cos_innovation_negg": [-0.16755807399749756, -0.1653653383255005, 0.32534727454185486], "cos_apical_negg": [0.046080008149147034, 0.0542776957154274, -0.1415373980998993], "cos_Ac_negg": [-0.1675722301006317, -0.16536720097064972, 0.32534730434417725], "r_norm": [1.2696797847747803, 1.4968675374984741, 3.008450984954834], "ldr": {"dL_sdil": -1.9025554656982422, "dL_bp": -1.9025554656982422, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 11.761016845703125, "cos_r_negg": [-0.01287061721086502, -0.06735193729400635, -0.0385841503739357], "cos_innovation_negg": [-0.18042925000190735, -0.16590286791324615, 0.28066006302833557], "cos_apical_negg": [-0.012870614416897297, -0.06735193729400635, -0.03858416527509689], "cos_Ac_negg": [-0.18043065071105957, -0.16587179899215698, 0.28066006302833557], "r_norm": [1.2144157886505127, 1.294379472732544, 2.8880460262298584]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.1135, "test_loss": 10.54988564453125}, {"step": 5700, "epoch": 12, "train_loss": 4.504752159118652, "cos_r_negg": [0.033178504556417465, -0.10059517621994019, 0.26030296087265015], "cos_innovation_negg": [-0.08326803147792816, -0.07829194515943527, 0.7997688055038452], "cos_apical_negg": [0.03317850083112717, -0.10059518367052078, 0.26030290126800537], "cos_Ac_negg": [-0.0832659900188446, -0.07830281555652618, 0.7997688055038452], "r_norm": [1.1017403602600098, 1.2252583503723145, 2.6190295219421387]}, {"step": 5800, "epoch": 12, "train_loss": 12.402713775634766, "cos_r_negg": [-0.02120404690504074, 0.1870843470096588, -0.366138219833374], "cos_innovation_negg": [-0.2049187272787094, -0.3321942687034607, 0.4586130380630493], "cos_apical_negg": [-0.021204041317105293, 0.18708436191082, -0.366138219833374], "cos_Ac_negg": [-0.20494437217712402, -0.3322075605392456, 0.4586130678653717], "r_norm": [1.277613639831543, 1.384798288345337, 3.035029172897339]}, {"step": 5900, "epoch": 12, "train_loss": 5.6544508934021, "cos_r_negg": [0.021466072648763657, 0.08863453567028046, -0.3640625774860382], "cos_innovation_negg": [-0.0702090784907341, -0.3271505832672119, 0.8775585889816284], "cos_apical_negg": [0.021466074511408806, 0.08863453567028046, -0.3640625476837158], "cos_Ac_negg": [-0.07023029029369354, -0.327148973941803, 0.8775585889816284], "r_norm": [1.154801845550537, 1.253584623336792, 2.8464016914367676]}, {"step": 6000, "epoch": 12, "train_loss": 12.991623878479004, "cos_r_negg": [-0.0341978445649147, 0.0753202959895134, 0.034765347838401794], "cos_innovation_negg": [-0.08287002146244049, -0.7208178043365479, 0.8287156224250793], "cos_apical_negg": [-0.034197840839624405, 0.0753202885389328, 0.03476536646485329], "cos_Ac_negg": [-0.0829041600227356, -0.7208141088485718, 0.8287156820297241], "r_norm": [1.3042879104614258, 1.389836072921753, 3.172173500061035], "ldr": {"dL_sdil": -2.888543128967285, "dL_bp": -2.888543128967285, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.1028, "test_loss": 13.79490341796875}, {"step": 6100, "epoch": 13, "train_loss": 15.776175498962402, "cos_r_negg": [0.049888595938682556, -0.15614530444145203, -0.27222248911857605], "cos_innovation_negg": [-0.1472887396812439, -0.36704009771347046, 0.538937509059906], "cos_apical_negg": [0.04988858848810196, -0.15614530444145203, -0.27222245931625366], "cos_Ac_negg": [-0.14728501439094543, -0.3670456111431122, 0.538937509059906], "r_norm": [1.1628806591033936, 1.1766693592071533, 2.8416147232055664]}, {"step": 6200, "epoch": 13, "train_loss": 16.496044158935547, "cos_r_negg": [0.014117803424596786, -0.3773772716522217, -0.5219522714614868], "cos_innovation_negg": [0.032156918197870255, 0.0008837850764393806, 0.2601567804813385], "cos_apical_negg": [0.014117800630629063, -0.3773772716522217, -0.5219523906707764], "cos_Ac_negg": [0.03220418840646744, 0.0008832784369587898, 0.2601567804813385], "r_norm": [1.2400844097137451, 1.2198686599731445, 3.034775733947754]}, {"step": 6300, "epoch": 13, "train_loss": 9.05245590209961, "cos_r_negg": [0.03672745078802109, -0.35884979367256165, -0.47191503643989563], "cos_innovation_negg": [-0.0891181007027626, 0.08326084911823273, 0.5798735618591309], "cos_apical_negg": [0.036727454513311386, -0.35884979367256165, -0.47191503643989563], "cos_Ac_negg": [-0.0891268253326416, 0.08326170593500137, 0.5798735618591309], "r_norm": [1.1116385459899902, 1.2412306070327759, 2.753157615661621]}, {"step": 6400, "epoch": 13, "train_loss": 6.07943868637085, "cos_r_negg": [0.011679553426802158, -0.17425653338432312, 0.03628295287489891], "cos_innovation_negg": [-0.0526568777859211, 0.059432484209537506, 0.6518957614898682], "cos_apical_negg": [0.011679550632834435, -0.17425653338432312, 0.036282941699028015], "cos_Ac_negg": [-0.05268612504005432, 0.05943384766578674, 0.6518958806991577], "r_norm": [1.1780421733856201, 1.2166333198547363, 2.965238332748413]}, {"step": 6500, "epoch": 13, "train_loss": 17.919506072998047, "cos_r_negg": [-0.020011212676763535, 0.067601777613163, -0.15226654708385468], "cos_innovation_negg": [-0.030089104548096657, -0.05048910528421402, 0.7527004480361938], "cos_apical_negg": [-0.020011218264698982, 0.0676017701625824, -0.1522665023803711], "cos_Ac_negg": [-0.03696989268064499, -0.05045740306377411, 0.7526998519897461], "r_norm": [1.249541997909546, 1.3109828233718872, 3.2425920963287354], "ldr": {"dL_sdil": -5.552864074707031, "dL_bp": -5.552864074707031, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.1135, "test_loss": 11.9120390625}, {"step": 6600, "epoch": 14, "train_loss": 17.50206756591797, "cos_r_negg": [0.03442477434873581, -0.32724928855895996, -0.31365150213241577], "cos_innovation_negg": [0.10971882194280624, -0.27853628993034363, 0.16275182366371155], "cos_apical_negg": [0.03442477434873581, -0.32724928855895996, -0.3136514723300934], "cos_Ac_negg": [0.10979901254177094, -0.27853628993034363, 0.16275030374526978], "r_norm": [1.269162654876709, 1.2095606327056885, 3.23329496383667]}, {"step": 6700, "epoch": 14, "train_loss": 10.686664581298828, "cos_r_negg": [0.03364798054099083, -0.3928048014640808, -0.22049809992313385], "cos_innovation_negg": [0.044101160019636154, 0.1364651769399643, 0.586864709854126], "cos_apical_negg": [0.033647987991571426, -0.3928048014640808, -0.22049811482429504], "cos_Ac_negg": [0.044156402349472046, 0.1364658772945404, 0.5868647694587708], "r_norm": [1.05025053024292, 1.0628093481063843, 2.7642130851745605]}, {"step": 6800, "epoch": 14, "train_loss": 17.495454788208008, "cos_r_negg": [0.07686907052993774, -0.3829383850097656, -0.4261513352394104], "cos_innovation_negg": [0.1924108862876892, 0.23426871001720428, 0.9896490573883057], "cos_apical_negg": [0.07686907052993774, -0.3829383850097656, -0.426151305437088], "cos_Ac_negg": [0.21469765901565552, 0.23420634865760803, 0.9896491765975952], "r_norm": [1.2391469478607178, 1.2442371845245361, 3.37134051322937]}, {"step": 6900, "epoch": 14, "train_loss": 19.883670806884766, "cos_r_negg": [0.0815495103597641, -0.3799760937690735, -0.4372183084487915], "cos_innovation_negg": [0.18566444516181946, 0.2155233919620514, 0.9861960411071777], "cos_apical_negg": [0.0815495103597641, -0.3799760937690735, -0.4372183680534363], "cos_Ac_negg": [0.1859016865491867, 0.21552357077598572, 0.9861960411071777], "r_norm": [1.2189700603485107, 1.2186787128448486, 3.3501100540161133]}, {"step": 7000, "epoch": 14, "train_loss": 12.419652938842773, "cos_r_negg": [0.015219427645206451, -0.2697301208972931, -0.19048017263412476], "cos_innovation_negg": [0.1253231018781662, 0.008636455982923508, 0.5253019332885742], "cos_apical_negg": [0.015219430439174175, -0.2697300910949707, -0.19048017263412476], "cos_Ac_negg": [0.12531673908233643, 0.00863754004240036, 0.5253018736839294], "r_norm": [1.1569595336914062, 1.1897273063659668, 3.1856937408447266], "ldr": {"dL_sdil": -2.832364082336426, "dL_bp": -2.832364082336426, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.098, "test_loss": 14.7233546875}], "final": {"test_acc": 0.098, "test_loss": 14.7233546875, "wall_s": 24.9082932472229, "cos_r_negg": [0.04708272963762283, -0.17973271012306213, 0.5334709882736206], "cos_innovation_negg": [-0.01665298268198967, 0.03750469163060188, 0.8107142448425293], "cos_apical_negg": [0.04708272963762283, -0.17973271012306213, 0.5334709882736206], "cos_Ac_negg": [-0.014648519456386566, 0.03750882297754288, 0.8107141256332397]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_matched_s4.json b/results/nuis_ctrl_v1_rho0p5_matched_s4.json new file mode 100644 index 0000000..8c8a8ba --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_matched_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "match_innovation_norm", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_matched_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.01129845343530178, 0.008651826530694962, -0.007999418303370476], "cos_innovation_negg": [-0.009800340980291367, 0.01927170902490616, 0.006490039173513651], "cos_apical_negg": [-0.01129845343530178, 0.008651826530694962, -0.007999418303370476], "cos_Ac_negg": [-0.009802713058888912, 0.01927042379975319, 0.006488661281764507], "r_norm": [4.8780083656311035, 4.877157211303711, 4.972771167755127], "ldr": {"dL_sdil": 0.04328036308288574, "dL_bp": -0.25147366523742676, "ratio": -0.17210694027155068}}, {"step": 100, "epoch": 0, "train_loss": 5.8541765213012695, "cos_r_negg": [0.011859158053994179, -0.2197123020887375, -0.3577398955821991], "cos_innovation_negg": [-0.0072913807816803455, -0.052386775612831116, -0.5470658540725708], "cos_apical_negg": [0.01185915432870388, -0.2197122871875763, -0.3577398657798767], "cos_Ac_negg": [-0.007299464195966721, -0.0523856058716774, -0.5470612049102783], "r_norm": [5.124417304992676, 5.268747806549072, 5.337373733520508]}, {"step": 200, "epoch": 0, "train_loss": 7.517156600952148, "cos_r_negg": [-0.0007674796506762505, -0.2255302369594574, -0.41825735569000244], "cos_innovation_negg": [-0.01766887493431568, 0.03300219029188156, -0.3543049991130829], "cos_apical_negg": [-0.0007674787193536758, -0.2255302369594574, -0.41825735569000244], "cos_Ac_negg": [-0.01773150824010372, 0.03299228847026825, -0.35431337356567383], "r_norm": [5.555302619934082, 5.636344909667969, 5.777846813201904]}, {"step": 300, "epoch": 0, "train_loss": 14.709612846374512, "cos_r_negg": [-0.05441128462553024, -0.013229746371507645, -0.5223036408424377], "cos_innovation_negg": [-0.05681538209319115, 0.05241924151778221, -0.4191136956214905], "cos_apical_negg": [-0.05441128462553024, -0.013229753822088242, -0.522303581237793], "cos_Ac_negg": [-0.05679492652416229, 0.05261079967021942, -0.4186246991157532], "r_norm": [5.259543418884277, 5.360218048095703, 5.377040386199951]}, {"step": 400, "epoch": 0, "train_loss": 13.243284225463867, "cos_r_negg": [0.0021957606077194214, -0.11069093644618988, -0.46535995602607727], "cos_innovation_negg": [-0.03861415758728981, -0.14249730110168457, -0.2690833806991577], "cos_apical_negg": [0.00219576433300972, -0.11069094389677048, -0.46535995602607727], "cos_Ac_negg": [-0.03862171620130539, -0.14250123500823975, -0.2690684199333191], "r_norm": [5.04359245300293, 5.089756965637207, 5.202094078063965]}, {"epoch_end": 0, "step": 469, "test_acc": 0.098, "test_loss": 11.74906416015625}, {"step": 500, "epoch": 1, "train_loss": 9.267217636108398, "cos_r_negg": [-0.07953251898288727, -0.21007049083709717, -0.4973289370536804], "cos_innovation_negg": [-0.07249242067337036, 0.005075649358332157, -0.5114177465438843], "cos_apical_negg": [-0.07953251898288727, -0.21007049083709717, -0.4973289370536804], "cos_Ac_negg": [-0.07248273491859436, 0.005073705688118935, -0.5114182829856873], "r_norm": [4.512006759643555, 4.719825267791748, 4.868722438812256], "ldr": {"dL_sdil": -2.430685043334961, "dL_bp": -2.430685043334961, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 14.977519989013672, "cos_r_negg": [-0.010794905945658684, 0.04714793711900711, -0.08304298669099808], "cos_innovation_negg": [0.026821741834282875, -0.04090297594666481, -0.1082586720585823], "cos_apical_negg": [-0.01079490501433611, 0.047147929668426514, -0.08304299414157867], "cos_Ac_negg": [0.026827357709407806, -0.04090838134288788, -0.10826852917671204], "r_norm": [4.290731430053711, 4.528001308441162, 4.737679958343506]}, {"step": 700, "epoch": 1, "train_loss": 9.509578704833984, "cos_r_negg": [0.0870501771569252, -0.009413352236151695, -0.2456909865140915], "cos_innovation_negg": [0.07207196950912476, 0.18434545397758484, -0.13110113143920898], "cos_apical_negg": [0.0870501846075058, -0.009413350373506546, -0.2456909716129303], "cos_Ac_negg": [0.07206246256828308, 0.18433862924575806, -0.13110670447349548], "r_norm": [4.123425483703613, 4.401701927185059, 4.547141075134277]}, {"step": 800, "epoch": 1, "train_loss": 11.306516647338867, "cos_r_negg": [-0.02568989433348179, 0.1021820455789566, -0.41206684708595276], "cos_innovation_negg": [-0.07593926787376404, -0.09788890928030014, -0.6020598411560059], "cos_apical_negg": [-0.02568989060819149, 0.1021820455789566, -0.41206681728363037], "cos_Ac_negg": [-0.07594180852174759, -0.09790432453155518, -0.6020537614822388], "r_norm": [3.778378963470459, 4.147512435913086, 4.202633857727051]}, {"step": 900, "epoch": 1, "train_loss": 11.810812950134277, "cos_r_negg": [-0.015443554148077965, 0.029148340225219727, -0.4459986686706543], "cos_innovation_negg": [-0.006619329564273357, -0.22585856914520264, -0.5746004581451416], "cos_apical_negg": [-0.015443547628819942, 0.02914833277463913, -0.4459986686706543], "cos_Ac_negg": [-0.006692266091704369, -0.22585037350654602, -0.5746238231658936], "r_norm": [3.9097304344177246, 4.300942897796631, 4.448230743408203]}, {"epoch_end": 1, "step": 938, "test_acc": 0.101, "test_loss": 7.963738671875}, {"step": 1000, "epoch": 2, "train_loss": 8.884724617004395, "cos_r_negg": [-0.039530590176582336, -0.08551079779863358, 0.6135400533676147], "cos_innovation_negg": [-0.04736698418855667, -0.11023949086666107, 0.5906074047088623], "cos_apical_negg": [-0.039530590176582336, -0.08551079779863358, 0.6135400533676147], "cos_Ac_negg": [-0.047403719276189804, -0.11028845608234406, 0.5905972719192505], "r_norm": [3.6757125854492188, 4.192357063293457, 4.424413681030273], "ldr": {"dL_sdil": -2.054088592529297, "dL_bp": -2.054088592529297, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 14.597789764404297, "cos_r_negg": [-0.04002337530255318, -0.01344760600477457, -0.2623630464076996], "cos_innovation_negg": [-0.08198900520801544, -0.3292337656021118, -0.19218310713768005], "cos_apical_negg": [-0.04002337530255318, -0.013447610661387444, -0.262363076210022], "cos_Ac_negg": [-0.08199483156204224, -0.3292442858219147, -0.19217684864997864], "r_norm": [3.2151031494140625, 3.6015751361846924, 3.772069215774536]}, {"step": 1200, "epoch": 2, "train_loss": 12.347772598266602, "cos_r_negg": [-0.04184480756521225, 0.029573867097496986, -0.454253226518631], "cos_innovation_negg": [-0.1306096613407135, -0.33464229106903076, -0.26127147674560547], "cos_apical_negg": [-0.04184481129050255, 0.029573867097496986, -0.454253226518631], "cos_Ac_negg": [-0.12359117716550827, -0.3321819603443146, -0.25948476791381836], "r_norm": [3.1707687377929688, 3.5867362022399902, 3.797882556915283]}, {"step": 1300, "epoch": 2, "train_loss": 7.944858551025391, "cos_r_negg": [-0.021507171913981438, -0.25699418783187866, 0.11335650086402893], "cos_innovation_negg": [-0.017513461410999298, 0.09577532857656479, 0.09815210103988647], "cos_apical_negg": [-0.02150716632604599, -0.25699418783187866, 0.11335647106170654], "cos_Ac_negg": [-0.01751203089952469, 0.09577464312314987, 0.09815260022878647], "r_norm": [2.7884225845336914, 3.1255950927734375, 3.325468063354492]}, {"step": 1400, "epoch": 2, "train_loss": 17.05021858215332, "cos_r_negg": [-0.015276666730642319, -0.05686268210411072, -0.5087875127792358], "cos_innovation_negg": [-0.003988954238593578, 0.09596237540245056, -0.04826423525810242], "cos_apical_negg": [-0.015276672318577766, -0.056862685829401016, -0.5087875127792358], "cos_Ac_negg": [-0.003919597715139389, 0.09464316070079803, -0.04777562990784645], "r_norm": [3.209172487258911, 3.6498122215270996, 3.9084768295288086]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.101, "test_loss": 11.65413310546875}, {"step": 1500, "epoch": 3, "train_loss": 11.066307067871094, "cos_r_negg": [-0.02007298171520233, -0.09805725514888763, -0.4332076907157898], "cos_innovation_negg": [-0.1192355826497078, -0.09998417645692825, -0.1597440540790558], "cos_apical_negg": [-0.020072977989912033, -0.09805724769830704, -0.433207631111145], "cos_Ac_negg": [-0.11802435666322708, -0.09912623465061188, -0.15952078998088837], "r_norm": [2.787120819091797, 3.2394533157348633, 3.539144277572632], "ldr": {"dL_sdil": -4.051599979400635, "dL_bp": -4.051599979400635, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 11.112384796142578, "cos_r_negg": [0.003439044114202261, -0.17804297804832458, -0.2650943994522095], "cos_innovation_negg": [0.006826162338256836, 0.18137340247631073, 0.08404278010129929], "cos_apical_negg": [0.0034390483051538467, -0.1780429631471634, -0.2650943994522095], "cos_Ac_negg": [0.006809420883655548, 0.18137305974960327, 0.0840434581041336], "r_norm": [2.9356188774108887, 3.378755807876587, 3.697742462158203]}, {"step": 1700, "epoch": 3, "train_loss": 14.974851608276367, "cos_r_negg": [-0.03673722967505455, -0.08957450091838837, 0.05323369801044464], "cos_innovation_negg": [0.03135113790631294, -0.15391230583190918, 0.5507566928863525], "cos_apical_negg": [-0.03673723340034485, -0.08957449346780777, 0.05323369801044464], "cos_Ac_negg": [0.03134145587682724, -0.15392333269119263, 0.550754189491272], "r_norm": [2.656477928161621, 2.9651055335998535, 3.2688050270080566]}, {"step": 1800, "epoch": 3, "train_loss": 10.519225120544434, "cos_r_negg": [-0.029512641951441765, 0.021447008475661278, -0.13711871206760406], "cos_innovation_negg": [-0.04361803084611893, -0.013496778905391693, -0.5173032283782959], "cos_apical_negg": [-0.029512640088796616, 0.02144699916243553, -0.13711872696876526], "cos_Ac_negg": [-0.04361617565155029, -0.013385454192757607, -0.517304003238678], "r_norm": [2.718841552734375, 3.16217041015625, 3.4883527755737305]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.1028, "test_loss": 15.07969111328125}, {"step": 1900, "epoch": 4, "train_loss": 11.25178050994873, "cos_r_negg": [0.02110508643090725, -0.029691100120544434, 0.29179835319519043], "cos_innovation_negg": [0.02706158719956875, 0.3323219418525696, -0.19874373078346252], "cos_apical_negg": [0.02110508643090725, -0.029691096395254135, 0.29179835319519043], "cos_Ac_negg": [0.027052119374275208, 0.3323248624801636, -0.19874323904514313], "r_norm": [2.4587879180908203, 2.8561155796051025, 3.2578701972961426]}, {"step": 2000, "epoch": 4, "train_loss": 10.440183639526367, "cos_r_negg": [-0.0377700999379158, -0.12785397469997406, -0.4010062515735626], "cos_innovation_negg": [-0.017192503437399864, 0.09235957264900208, 0.14360617101192474], "cos_apical_negg": [-0.0377700999379158, -0.12785397469997406, -0.4010062515735626], "cos_Ac_negg": [-0.017174983397126198, 0.09244447946548462, 0.14362017810344696], "r_norm": [2.2871251106262207, 2.7045998573303223, 3.265491485595703], "ldr": {"dL_sdil": -2.0468292236328125, "dL_bp": -2.0468292236328125, "ratio": 1.0}}, {"step": 2100, "epoch": 4, "train_loss": 9.651744842529297, "cos_r_negg": [0.02698163315653801, -0.1698457896709442, -0.4057127833366394], "cos_innovation_negg": [0.0902349203824997, 0.0648086741566658, 0.20928771793842316], "cos_apical_negg": [0.02698163501918316, -0.1698457896709442, -0.4057127833366394], "cos_Ac_negg": [0.09113830327987671, 0.06552675366401672, 0.20948526263237], "r_norm": [2.600207805633545, 2.962268352508545, 3.4584484100341797]}, {"step": 2200, "epoch": 4, "train_loss": 9.51034164428711, "cos_r_negg": [-0.029838094487786293, -0.023607686161994934, 0.5379142165184021], "cos_innovation_negg": [0.0011311490088701248, 0.04019015282392502, 0.7120953798294067], "cos_apical_negg": [-0.029838092625141144, -0.023607686161994934, 0.5379142165184021], "cos_Ac_negg": [0.0011341515928506851, 0.040197186172008514, 0.7120956778526306], "r_norm": [2.1194517612457275, 2.504669666290283, 3.0483927726745605]}, {"step": 2300, "epoch": 4, "train_loss": 11.1998291015625, "cos_r_negg": [-0.0303325317800045, -0.16468754410743713, -0.3945731520652771], "cos_innovation_negg": [0.03663358464837074, 0.07643117755651474, 0.2164628803730011], "cos_apical_negg": [-0.030332528054714203, -0.16468754410743713, -0.3945731520652771], "cos_Ac_negg": [0.03664188086986542, 0.07645708322525024, 0.21646468341350555], "r_norm": [2.315669536590576, 2.630915880203247, 3.1710197925567627]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.0982, "test_loss": 11.56928701171875}, {"step": 2400, "epoch": 5, "train_loss": 15.464273452758789, "cos_r_negg": [-0.022517157718539238, -0.06016632914543152, -0.2213427722454071], "cos_innovation_negg": [0.009990440681576729, 0.10562947392463684, -0.07143792510032654], "cos_apical_negg": [-0.022517159581184387, -0.06016632542014122, -0.22134274244308472], "cos_Ac_negg": [0.007077903486788273, 0.12128163874149323, -0.07162907719612122], "r_norm": [2.319413661956787, 2.812208890914917, 3.1901986598968506]}, {"step": 2500, "epoch": 5, "train_loss": 15.781010627746582, "cos_r_negg": [0.006515308283269405, 0.03680349886417389, 0.17628473043441772], "cos_innovation_negg": [-0.039479028433561325, -0.023504361510276794, 0.3867316246032715], "cos_apical_negg": [0.006515307351946831, 0.03680350258946419, 0.1762847602367401], "cos_Ac_negg": [-0.03948182612657547, -0.023458516225218773, 0.3867321312427521], "r_norm": [2.242642641067505, 2.6207966804504395, 3.155750274658203], "ldr": {"dL_sdil": -2.5270824432373047, "dL_bp": -2.5270824432373047, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 12.724416732788086, "cos_r_negg": [-0.009419533424079418, -0.057463571429252625, -0.2334344983100891], "cos_innovation_negg": [-0.05791371315717697, -0.1367834508419037, 0.4078209400177002], "cos_apical_negg": [-0.00941953994333744, -0.05746357887983322, -0.23343446850776672], "cos_Ac_negg": [-0.05791257321834564, -0.13678792119026184, 0.4078206419944763], "r_norm": [1.962188482284546, 2.3691956996917725, 2.9482617378234863]}, {"step": 2700, "epoch": 5, "train_loss": 9.730428695678711, "cos_r_negg": [0.0072060986422002316, -0.14397896826267242, 0.32661041617393494], "cos_innovation_negg": [0.012847809121012688, 0.23192207515239716, 0.5787216424942017], "cos_apical_negg": [0.007206094451248646, -0.14397895336151123, 0.32661041617393494], "cos_Ac_negg": [0.01285066269338131, 0.23192547261714935, 0.5787215232849121], "r_norm": [1.8642280101776123, 2.232654571533203, 2.835360050201416]}, {"step": 2800, "epoch": 5, "train_loss": 11.136555671691895, "cos_r_negg": [0.0017272725235670805, -0.06760498881340027, 0.07126668095588684], "cos_innovation_negg": [0.013942636549472809, 0.07325094938278198, 0.5194573998451233], "cos_apical_negg": [0.0017272711265832186, -0.06760498881340027, 0.07126669585704803], "cos_Ac_negg": [0.013938605785369873, 0.07326620817184448, 0.5194574594497681], "r_norm": [1.8120512962341309, 2.11643648147583, 2.732422351837158]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.0892, "test_loss": 9.65150244140625}, {"step": 2900, "epoch": 6, "train_loss": 8.220255851745605, "cos_r_negg": [0.021356161683797836, -0.10543379187583923, 0.39407429099082947], "cos_innovation_negg": [0.06694730371236801, 0.10491691529750824, 0.29907649755477905], "cos_apical_negg": [0.021356161683797836, -0.10543379187583923, 0.3940742611885071], "cos_Ac_negg": [0.06694920361042023, 0.10499151051044464, 0.2990766763687134], "r_norm": [1.7774038314819336, 2.08298659324646, 2.789957046508789]}, {"step": 3000, "epoch": 6, "train_loss": 10.099320411682129, "cos_r_negg": [0.01722574047744274, -0.12536576390266418, -0.4060935378074646], "cos_innovation_negg": [0.022347591817378998, 0.15969829261302948, 0.25333529710769653], "cos_apical_negg": [0.01722574047744274, -0.125365749001503, -0.4060935378074646], "cos_Ac_negg": [0.02230764552950859, 0.1597033143043518, 0.25333642959594727], "r_norm": [1.7957839965820312, 2.1143641471862793, 3.030028820037842], "ldr": {"dL_sdil": -2.4251184463500977, "dL_bp": -2.4251184463500977, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 11.909306526184082, "cos_r_negg": [-0.02589208260178566, 0.07551902532577515, -0.22723469138145447], "cos_innovation_negg": [-0.08696787059307098, -0.13182185590267181, 0.7523332834243774], "cos_apical_negg": [-0.02589208073914051, 0.07551901042461395, -0.22723469138145447], "cos_Ac_negg": [-0.08697381615638733, -0.13182967901229858, 0.7523332834243774], "r_norm": [1.8209266662597656, 2.16239070892334, 2.90501070022583]}, {"step": 3200, "epoch": 6, "train_loss": 18.356176376342773, "cos_r_negg": [-0.013865020126104355, -0.06947062909603119, 0.3276683986186981], "cos_innovation_negg": [0.043771419674158096, 0.12700802087783813, 0.6365551948547363], "cos_apical_negg": [-0.013865020126104355, -0.06947062909603119, 0.3276683986186981], "cos_Ac_negg": [0.04369744658470154, 0.12648585438728333, 0.6365549564361572], "r_norm": [1.8338781595230103, 2.2880051136016846, 3.1727118492126465]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.1009, "test_loss": 17.4827533203125}, {"step": 3300, "epoch": 7, "train_loss": 13.18510913848877, "cos_r_negg": [-0.0316130705177784, 0.1357533037662506, 0.009787240996956825], "cos_innovation_negg": [0.011263503693044186, -0.13506253063678741, 0.689012885093689], "cos_apical_negg": [-0.0316130667924881, 0.1357533037662506, 0.009787237271666527], "cos_Ac_negg": [0.011266272515058517, -0.1350746750831604, 0.6890128254890442], "r_norm": [1.599574327468872, 1.8772563934326172, 2.674494743347168]}, {"step": 3400, "epoch": 7, "train_loss": 21.204906463623047, "cos_r_negg": [-0.03684203699231148, -0.09438958764076233, 0.13558955490589142], "cos_innovation_negg": [-0.015204884111881256, -0.29622703790664673, 0.701994001865387], "cos_apical_negg": [-0.03684203699231148, -0.09438958764076233, 0.13558955490589142], "cos_Ac_negg": [-0.015198917128145695, -0.29621008038520813, 0.701994001865387], "r_norm": [1.675079584121704, 1.909162163734436, 2.802018880844116]}, {"step": 3500, "epoch": 7, "train_loss": 8.05005931854248, "cos_r_negg": [0.025616846978664398, -0.06617026031017303, -0.26765990257263184], "cos_innovation_negg": [-0.018649492412805557, -0.10789841413497925, 0.851392388343811], "cos_apical_negg": [0.025616848841309547, -0.06617026776075363, -0.26765990257263184], "cos_Ac_negg": [-0.018653957173228264, -0.10781063139438629, 0.8513922691345215], "r_norm": [1.5803498029708862, 1.8384654521942139, 2.68916916847229], "ldr": {"dL_sdil": -1.5574464797973633, "dL_bp": -1.5574464797973633, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 10.90673828125, "cos_r_negg": [-0.04887796565890312, -0.06822121888399124, -0.40664350986480713], "cos_innovation_negg": [-0.0952325165271759, -0.1707383096218109, 0.8282569646835327], "cos_apical_negg": [-0.04887796938419342, -0.06822122633457184, -0.40664350986480713], "cos_Ac_negg": [-0.09523377567529678, -0.17073523998260498, 0.8282568454742432], "r_norm": [1.5510720014572144, 1.7978637218475342, 2.663027048110962]}, {"step": 3700, "epoch": 7, "train_loss": 15.147183418273926, "cos_r_negg": [0.00045548751950263977, -0.11392904818058014, -0.3510252833366394], "cos_innovation_negg": [-0.07126955687999725, -0.3411969244480133, 0.6208035349845886], "cos_apical_negg": [0.0004554903134703636, -0.11392904818058014, -0.3510252833366394], "cos_Ac_negg": [-0.07125917077064514, -0.34114325046539307, 0.6208035349845886], "r_norm": [1.556649088859558, 1.7780513763427734, 2.8818283081054688]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.0892, "test_loss": 6.903949658203125}, {"step": 3800, "epoch": 8, "train_loss": 6.643497467041016, "cos_r_negg": [-0.02152605913579464, -0.10375601053237915, -0.23378929495811462], "cos_innovation_negg": [-0.06955479085445404, -0.06414562463760376, 0.7090858221054077], "cos_apical_negg": [-0.02152606099843979, -0.10375600308179855, -0.23378928005695343], "cos_Ac_negg": [-0.06956103444099426, -0.06412623077630997, 0.7090858221054077], "r_norm": [1.4977200031280518, 1.6852657794952393, 2.6745219230651855]}, {"step": 3900, "epoch": 8, "train_loss": 11.924671173095703, "cos_r_negg": [-0.0012499094009399414, 0.08291405439376831, 0.05258265137672424], "cos_innovation_negg": [-0.0723571628332138, 0.0009395778179168701, 0.7132940292358398], "cos_apical_negg": [-0.0012499038130044937, 0.08291403949260712, 0.05258265882730484], "cos_Ac_negg": [-0.0723511129617691, 0.000929027795791626, 0.7132940292358398], "r_norm": [1.6799412965774536, 1.8777098655700684, 3.105379581451416]}, {"step": 4000, "epoch": 8, "train_loss": 5.951869010925293, "cos_r_negg": [-0.023763399571180344, -0.12959745526313782, -0.0574309416115284], "cos_innovation_negg": [0.008056455291807652, -0.20656552910804749, 0.6868358850479126], "cos_apical_negg": [-0.023763395845890045, -0.12959745526313782, -0.057430945336818695], "cos_Ac_negg": [0.008054409176111221, -0.20657196640968323, 0.6868358850479126], "r_norm": [1.3941377401351929, 1.588182806968689, 2.5991177558898926], "ldr": {"dL_sdil": -2.3911619186401367, "dL_bp": -2.3911614418029785, "ratio": 1.000000199416547}}, {"step": 4100, "epoch": 8, "train_loss": 15.187675476074219, "cos_r_negg": [-0.04170889034867287, -0.041121289134025574, -0.09154659509658813], "cos_innovation_negg": [-0.20098094642162323, -0.30104750394821167, 0.8014010787010193], "cos_apical_negg": [-0.04170889034867287, -0.04112128168344498, -0.09154658764600754], "cos_Ac_negg": [-0.20099011063575745, -0.30104202032089233, 0.8014011383056641], "r_norm": [1.502905249595642, 1.6867352724075317, 2.754103660583496]}, {"step": 4200, "epoch": 8, "train_loss": 9.053960800170898, "cos_r_negg": [0.04265213757753372, -0.09080851078033447, 0.024620793759822845], "cos_innovation_negg": [0.03838331624865532, 0.18795913457870483, 0.6641244888305664], "cos_apical_negg": [0.04265213385224342, -0.09080851078033447, 0.024620765820145607], "cos_Ac_negg": [0.03836546838283539, 0.18793150782585144, 0.6641243100166321], "r_norm": [1.6065177917480469, 1.77130126953125, 3.0752768516540527]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.098, "test_loss": 7.322003125}, {"step": 4300, "epoch": 9, "train_loss": 9.786920547485352, "cos_r_negg": [-0.019598793238401413, -0.19801032543182373, -0.2530418932437897], "cos_innovation_negg": [0.0384855717420578, 0.24911805987358093, 0.7266221046447754], "cos_apical_negg": [-0.019598783925175667, -0.19801032543182373, -0.25304192304611206], "cos_Ac_negg": [0.03852107375860214, 0.2480456531047821, 0.7266221046447754], "r_norm": [1.5966852903366089, 1.7460834980010986, 3.0226521492004395]}, {"step": 4400, "epoch": 9, "train_loss": 8.033329963684082, "cos_r_negg": [-0.039990052580833435, 0.11848752945661545, 0.3446117639541626], "cos_innovation_negg": [0.03951078653335571, 0.2667197287082672, 0.7486597299575806], "cos_apical_negg": [-0.03999006003141403, 0.11848751455545425, 0.3446117043495178], "cos_Ac_negg": [0.039519526064395905, 0.26671546697616577, 0.7486597299575806], "r_norm": [1.5250786542892456, 1.7146329879760742, 2.9164438247680664]}, {"step": 4500, "epoch": 9, "train_loss": 9.973971366882324, "cos_r_negg": [0.05105758085846901, -0.24648331105709076, 0.08119643479585648], "cos_innovation_negg": [0.030349545180797577, 0.10909681022167206, 0.7348092794418335], "cos_apical_negg": [0.05105758085846901, -0.24648331105709076, 0.08119641989469528], "cos_Ac_negg": [0.03032403439283371, 0.10909319669008255, 0.7348092794418335], "r_norm": [1.5895898342132568, 1.7133036851882935, 3.1713881492614746], "ldr": {"dL_sdil": -2.658766746520996, "dL_bp": -2.658766746520996, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 11.34526252746582, "cos_r_negg": [0.052056387066841125, -0.11308931559324265, 0.15062031149864197], "cos_innovation_negg": [0.07969589531421661, 0.0874897837638855, 0.8855361938476562], "cos_apical_negg": [0.05205639451742172, -0.11308931559324265, 0.15062028169631958], "cos_Ac_negg": [0.0796777680516243, 0.08753896504640579, 0.8855360746383667], "r_norm": [1.5672414302825928, 1.68569016456604, 3.1695499420166016]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.1028, "test_loss": 5.812430810546875}, {"step": 4700, "epoch": 10, "train_loss": 6.853944778442383, "cos_r_negg": [0.015469450503587723, -0.030090004205703735, 0.10213199257850647], "cos_innovation_negg": [0.07592116296291351, -0.19599664211273193, 0.8309423923492432], "cos_apical_negg": [0.015469452366232872, -0.030090004205703735, 0.10213200747966766], "cos_Ac_negg": [0.07592664659023285, -0.19599175453186035, 0.8309423923492432], "r_norm": [1.2418698072433472, 1.3654282093048096, 2.5544018745422363]}, {"step": 4800, "epoch": 10, "train_loss": 7.203009128570557, "cos_r_negg": [0.05201717093586922, -0.1968211531639099, 0.07510508596897125], "cos_innovation_negg": [0.026504985988140106, 0.15712064504623413, 0.9333200454711914], "cos_apical_negg": [0.052017174661159515, -0.1968211531639099, 0.07510510087013245], "cos_Ac_negg": [0.02649352326989174, 0.15710905194282532, 0.9333200454711914], "r_norm": [1.4653611183166504, 1.6132922172546387, 3.019134759902954]}, {"step": 4900, "epoch": 10, "train_loss": 10.213006019592285, "cos_r_negg": [0.02470514550805092, -0.045736514031887054, 0.23600004613399506], "cos_innovation_negg": [0.02771253138780594, 0.005600770935416222, 0.6410759091377258], "cos_apical_negg": [0.02470514178276062, -0.04573650658130646, 0.23600006103515625], "cos_Ac_negg": [0.027369946241378784, 0.005620865151286125, 0.6410759091377258], "r_norm": [1.4204668998718262, 1.6538618803024292, 3.190216541290283]}, {"step": 5000, "epoch": 10, "train_loss": 7.825192451477051, "cos_r_negg": [0.011988598853349686, -0.06839568167924881, 0.21490582823753357], "cos_innovation_negg": [0.042516279965639114, -0.09857931733131409, 0.7615839242935181], "cos_apical_negg": [0.011988595128059387, -0.06839567422866821, 0.21490582823753357], "cos_Ac_negg": [0.04253878444433212, -0.09857695549726486, 0.7615839242935181], "r_norm": [1.3786802291870117, 1.6490967273712158, 2.8277299404144287], "ldr": {"dL_sdil": -2.1439099311828613, "dL_bp": -2.1439099311828613, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 7.116961479187012, "cos_r_negg": [0.013926435261964798, -0.2839219570159912, -0.22931286692619324], "cos_innovation_negg": [0.001002265140414238, -0.2436596155166626, 0.8811333179473877], "cos_apical_negg": [0.013926437124609947, -0.2839219570159912, -0.22931286692619324], "cos_Ac_negg": [0.0011710533872246742, -0.2436147779226303, 0.8811327815055847], "r_norm": [1.4912426471710205, 1.5809907913208008, 3.143693208694458]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.1009, "test_loss": 12.3547849609375}, {"step": 5200, "epoch": 11, "train_loss": 13.147964477539062, "cos_r_negg": [-0.012514306232333183, -0.01933658868074417, 0.07910460233688354], "cos_innovation_negg": [-0.07245813310146332, 0.028523260727524757, 0.49874839186668396], "cos_apical_negg": [-0.012514311820268631, -0.019336584955453873, 0.07910459488630295], "cos_Ac_negg": [-0.07248740643262863, 0.02852851338684559, 0.4987483620643616], "r_norm": [1.404446005821228, 1.715101718902588, 2.95341157913208]}, {"step": 5300, "epoch": 11, "train_loss": 8.042003631591797, "cos_r_negg": [-0.04400178790092468, 0.15651726722717285, 0.43182075023651123], "cos_innovation_negg": [0.008472003042697906, 0.07192736864089966, 0.760715901851654], "cos_apical_negg": [-0.044001780450344086, 0.15651726722717285, 0.43182075023651123], "cos_Ac_negg": [0.008475187234580517, 0.07194077223539352, 0.760715901851654], "r_norm": [1.2975459098815918, 1.46932053565979, 3.03129243850708]}, {"step": 5400, "epoch": 11, "train_loss": 14.67538833618164, "cos_r_negg": [0.07488811016082764, -0.611284613609314, -0.03292857110500336], "cos_innovation_negg": [0.0623897910118103, -0.1497766375541687, 0.9646369218826294], "cos_apical_negg": [0.07488811761140823, -0.611284613609314, -0.03292855620384216], "cos_Ac_negg": [0.06242059916257858, -0.14913736283779144, 0.9646369218826294], "r_norm": [1.2761456966400146, 1.4081602096557617, 2.8667917251586914]}, {"step": 5500, "epoch": 11, "train_loss": 13.10124397277832, "cos_r_negg": [0.04806559532880783, -0.5321519374847412, -0.43347206711769104], "cos_innovation_negg": [-0.0356353260576725, -0.27107638120651245, 0.7018305063247681], "cos_apical_negg": [0.04806559160351753, -0.5321519374847412, -0.4334720969200134], "cos_Ac_negg": [-0.0355859212577343, -0.27106714248657227, 0.7018305063247681], "r_norm": [1.3927748203277588, 1.4736299514770508, 3.0802412033081055], "ldr": {"dL_sdil": -2.2028512954711914, "dL_bp": -2.2028512954711914, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 12.440616607666016, "cos_r_negg": [0.02137741446495056, -0.21128374338150024, -0.025200016796588898], "cos_innovation_negg": [0.011442672461271286, 0.08184020221233368, 0.8471606969833374], "cos_apical_negg": [0.02137741446495056, -0.21128374338150024, -0.02520000748336315], "cos_Ac_negg": [0.011423293501138687, 0.08184285461902618, 0.8471606373786926], "r_norm": [1.340846300125122, 1.4272390604019165, 3.1976871490478516]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.0958, "test_loss": 17.180212109375}, {"step": 5700, "epoch": 12, "train_loss": 8.56231689453125, "cos_r_negg": [-0.0072683049365878105, -0.0635027214884758, 0.13623985648155212], "cos_innovation_negg": [0.09627002477645874, 0.022203674539923668, 0.6805067658424377], "cos_apical_negg": [-0.0072683123871684074, -0.0635027140378952, 0.13623985648155212], "cos_Ac_negg": [0.09627658128738403, 0.022205056622624397, 0.6805067658424377], "r_norm": [1.1378693580627441, 1.2611987590789795, 2.694638252258301]}, {"step": 5800, "epoch": 12, "train_loss": 13.849864959716797, "cos_r_negg": [0.06501591205596924, -0.09321749210357666, 0.0991235300898552], "cos_innovation_negg": [0.07016571611166, -0.021892592310905457, 0.583567202091217], "cos_apical_negg": [0.06501591205596924, -0.09321749210357666, 0.09912355244159698], "cos_Ac_negg": [0.07015561312437057, -0.02187947928905487, 0.583567202091217], "r_norm": [1.1757150888442993, 1.3544279336929321, 2.7343907356262207]}, {"step": 5900, "epoch": 12, "train_loss": 11.105947494506836, "cos_r_negg": [0.05209483578801155, -0.1640486866235733, 0.12589792907238007], "cos_innovation_negg": [0.12532930076122284, -0.021641336381435394, 0.8329748511314392], "cos_apical_negg": [0.05209483206272125, -0.1640486866235733, 0.12589794397354126], "cos_Ac_negg": [0.12534697353839874, -0.021634595468640327, 0.832974910736084], "r_norm": [1.1829010248184204, 1.2659488916397095, 2.935713291168213]}, {"step": 6000, "epoch": 12, "train_loss": 10.433158874511719, "cos_r_negg": [0.014796750620007515, -0.07796542346477509, 0.09792005270719528], "cos_innovation_negg": [0.020694822072982788, 0.05126030743122101, 0.795454740524292], "cos_apical_negg": [0.014796756207942963, -0.07796542346477509, 0.09792004525661469], "cos_Ac_negg": [0.02072843909263611, 0.05124151334166527, 0.7954549789428711], "r_norm": [1.2778316736221313, 1.351291537284851, 3.2066588401794434], "ldr": {"dL_sdil": -2.0467586517333984, "dL_bp": -2.0467586517333984, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.1032, "test_loss": 7.150468603515625}, {"step": 6100, "epoch": 13, "train_loss": 5.191860675811768, "cos_r_negg": [-0.04105672985315323, 0.007905213162302971, 0.19615964591503143], "cos_innovation_negg": [0.027023103088140488, 0.1204105019569397, 0.7754322290420532], "cos_apical_negg": [-0.041056737303733826, 0.007905209437012672, 0.19615964591503143], "cos_Ac_negg": [0.027046678587794304, 0.12040675431489944, 0.7754322290420532], "r_norm": [1.2143007516860962, 1.499632716178894, 2.9527604579925537]}, {"step": 6200, "epoch": 13, "train_loss": 9.029670715332031, "cos_r_negg": [-0.006353901699185371, -0.4460465610027313, -0.25975489616394043], "cos_innovation_negg": [-0.018985139206051826, -0.19720140099525452, 0.8650809526443481], "cos_apical_negg": [-0.006353903561830521, -0.4460465908050537, -0.25975489616394043], "cos_Ac_negg": [-0.0189388245344162, -0.19720467925071716, 0.8650810122489929], "r_norm": [1.3595330715179443, 1.4422768354415894, 3.255006790161133]}, {"step": 6300, "epoch": 13, "train_loss": 15.838714599609375, "cos_r_negg": [0.05555335059762001, -0.6325956583023071, -0.11965794861316681], "cos_innovation_negg": [0.2120371013879776, 0.3714461922645569, 0.9151303172111511], "cos_apical_negg": [0.05555334687232971, -0.6325956583023071, -0.11965794861316681], "cos_Ac_negg": [0.21200968325138092, 0.3714919686317444, 0.9151303172111511], "r_norm": [1.1652469635009766, 1.2346633672714233, 2.8637773990631104]}, {"step": 6400, "epoch": 13, "train_loss": 7.844067096710205, "cos_r_negg": [-0.05761849880218506, 0.21298682689666748, 0.3719564974308014], "cos_innovation_negg": [0.02575295977294445, 0.059048887342214584, 0.6961765289306641], "cos_apical_negg": [-0.057618506252765656, 0.21298682689666748, 0.3719564974308014], "cos_Ac_negg": [0.025752389803528786, 0.05904940143227577, 0.6961765289306641], "r_norm": [1.2223587036132812, 1.2979081869125366, 3.1317667961120605]}, {"step": 6500, "epoch": 13, "train_loss": 12.947637557983398, "cos_r_negg": [-0.03062296099960804, -0.12986260652542114, -0.07585766166448593], "cos_innovation_negg": [0.045764174312353134, 0.05654685199260712, 0.7535033226013184], "cos_apical_negg": [-0.03062296099960804, -0.12986260652542114, -0.07585765421390533], "cos_Ac_negg": [0.04517292231321335, 0.05654573440551758, 0.7535032033920288], "r_norm": [1.2442855834960938, 1.2960867881774902, 3.330897331237793], "ldr": {"dL_sdil": -3.8242359161376953, "dL_bp": -3.8242359161376953, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.1135, "test_loss": 15.10237890625}, {"step": 6600, "epoch": 14, "train_loss": 9.533498764038086, "cos_r_negg": [0.0014785490930080414, 0.059834618121385574, 0.21069705486297607], "cos_innovation_negg": [0.010342145338654518, 0.021155379712581635, 0.871597409248352], "cos_apical_negg": [0.0014785528182983398, 0.05983462184667587, 0.21069708466529846], "cos_Ac_negg": [0.010340411216020584, 0.02115510404109955, 0.8715974688529968], "r_norm": [1.069385290145874, 1.1482353210449219, 2.7854769229888916]}, {"step": 6700, "epoch": 14, "train_loss": 15.817525863647461, "cos_r_negg": [0.002576893661171198, -0.30749720335006714, -0.28639906644821167], "cos_innovation_negg": [0.032748255878686905, -0.09695050865411758, 0.7067950963973999], "cos_apical_negg": [0.0025768987834453583, -0.30749717354774475, -0.2863990366458893], "cos_Ac_negg": [0.032717593014240265, -0.09695038199424744, 0.7067950367927551], "r_norm": [1.216059923171997, 1.2632243633270264, 3.321539878845215]}, {"step": 6800, "epoch": 14, "train_loss": 9.432249069213867, "cos_r_negg": [0.007714568637311459, -0.18908476829528809, 0.1124211847782135], "cos_innovation_negg": [-0.055605608969926834, 0.043440189212560654, 0.8781342506408691], "cos_apical_negg": [0.007714565843343735, -0.18908476829528809, 0.11242115497589111], "cos_Ac_negg": [-0.055614031851291656, 0.04344010353088379, 0.8781342506408691], "r_norm": [1.101890206336975, 1.3312838077545166, 2.916893720626831]}, {"step": 6900, "epoch": 14, "train_loss": 13.774052619934082, "cos_r_negg": [-0.027112925425171852, -0.14012667536735535, 0.002092570997774601], "cos_innovation_negg": [-0.036226239055395126, 0.027317769825458527, 0.6915818452835083], "cos_apical_negg": [-0.027112923562526703, -0.14012667536735535, 0.0020925812423229218], "cos_Ac_negg": [-0.03619460016489029, 0.027317708358168602, 0.6915818452835083], "r_norm": [1.126561164855957, 1.207656979560852, 3.091043472290039]}, {"step": 7000, "epoch": 14, "train_loss": 7.675116539001465, "cos_r_negg": [0.05577746778726578, -0.3758794367313385, -0.19762195646762848], "cos_innovation_negg": [0.09373458474874496, 0.021276826038956642, 0.8586548566818237], "cos_apical_negg": [0.05577746778726578, -0.3758794665336609, -0.19762194156646729], "cos_Ac_negg": [0.0937395766377449, 0.021276796236634254, 0.8586548566818237], "r_norm": [0.9844045639038086, 1.0432591438293457, 2.6779537200927734], "ldr": {"dL_sdil": -0.9830341339111328, "dL_bp": -0.9830341339111328, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.098, "test_loss": 19.8634103515625}], "final": {"test_acc": 0.098, "test_loss": 19.8634103515625, "wall_s": 25.657039403915405, "cos_r_negg": [-0.0023339702747762203, -0.37329721450805664, -0.18468943238258362], "cos_innovation_negg": [-0.1657343953847885, 0.35242760181427, 0.501003623008728], "cos_apical_negg": [-0.0023339674808084965, -0.37329721450805664, -0.18468943238258362], "cos_Ac_negg": [-0.16574478149414062, 0.35242760181427, 0.5010035634040833]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_raw_s0.json b/results/nuis_ctrl_v1_rho0p5_raw_s0.json new file mode 100644 index 0000000..90a16b7 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_raw_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_raw_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.011117599904537201, -0.025546539574861526, 0.02163308672606945], "cos_innovation_negg": [-0.011871755123138428, -0.029108529910445213, 0.031950101256370544], "cos_apical_negg": [-0.011117599904537201, -0.025546539574861526, 0.02163308672606945], "cos_Ac_negg": [-0.011872358620166779, -0.029109783470630646, 0.031949400901794434], "r_norm": [8.172515869140625, 6.909910202026367, 6.500560760498047], "ldr": {"dL_sdil": 0.06155729293823242, "dL_bp": -0.18675637245178223, "ratio": -0.32961281122616376}}, {"step": 100, "epoch": 0, "train_loss": 6.401466369628906, "cos_r_negg": [-0.071133553981781, -0.2756350040435791, 0.23067882657051086], "cos_innovation_negg": [-0.10466733574867249, -0.34171411395072937, 0.4481695890426636], "cos_apical_negg": [-0.071133553981781, -0.2756350040435791, 0.23067882657051086], "cos_Ac_negg": [-0.10466187447309494, -0.3417147397994995, 0.4481753408908844], "r_norm": [10.948808670043945, 9.91490364074707, 9.659982681274414]}, {"step": 200, "epoch": 0, "train_loss": 16.67608070373535, "cos_r_negg": [-0.02049953117966652, -0.015601231716573238, -0.35598623752593994], "cos_innovation_negg": [-0.04517069831490517, -0.13602283596992493, -0.3602122962474823], "cos_apical_negg": [-0.02049953117966652, -0.015601231716573238, -0.35598623752593994], "cos_Ac_negg": [-0.045176245272159576, -0.13600605726242065, -0.3602127432823181], "r_norm": [10.548009872436523, 9.865960121154785, 9.979389190673828]}, {"step": 300, "epoch": 0, "train_loss": 13.683351516723633, "cos_r_negg": [-0.033369533717632294, -0.07613806426525116, -0.17397639155387878], "cos_innovation_negg": [-0.05950778350234032, -0.20307549834251404, 0.2607564628124237], "cos_apical_negg": [-0.033369533717632294, -0.07613806426525116, -0.17397639155387878], "cos_Ac_negg": [-0.05950779467821121, -0.20305851101875305, 0.26075488328933716], "r_norm": [10.32884693145752, 10.088122367858887, 9.798168182373047]}, {"step": 400, "epoch": 0, "train_loss": 10.192171096801758, "cos_r_negg": [-0.06079820543527603, -0.02414802461862564, -0.13587599992752075], "cos_innovation_negg": [-0.09456092119216919, -0.0638977587223053, 0.030106540769338608], "cos_apical_negg": [-0.06079820543527603, -0.02414802461862564, -0.13587599992752075], "cos_Ac_negg": [-0.0945574939250946, -0.06389879435300827, 0.03010324202477932], "r_norm": [10.427574157714844, 9.688739776611328, 9.38677978515625]}, {"epoch_end": 0, "step": 469, "test_acc": 0.101, "test_loss": 14.72414833984375}, {"step": 500, "epoch": 1, "train_loss": 9.624497413635254, "cos_r_negg": [0.024017971009016037, -0.10674364119768143, -0.1719636619091034], "cos_innovation_negg": [0.02288432978093624, 0.0907701700925827, 0.32318681478500366], "cos_apical_negg": [0.024017971009016037, -0.10674364119768143, -0.1719636619091034], "cos_Ac_negg": [0.0231236070394516, 0.09108787775039673, 0.32309696078300476], "r_norm": [9.645282745361328, 9.428837776184082, 9.333595275878906], "ldr": {"dL_sdil": -2.213644504547119, "dL_bp": -2.213644504547119, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 8.633424758911133, "cos_r_negg": [-0.03392370417714119, -0.0528486967086792, 0.21150174736976624], "cos_innovation_negg": [-0.08722939342260361, -0.08993001282215118, 0.5808777213096619], "cos_apical_negg": [-0.03392370417714119, -0.0528486967086792, 0.21150174736976624], "cos_Ac_negg": [-0.08722865581512451, -0.08995106816291809, 0.5808756351470947], "r_norm": [9.350034713745117, 9.030904769897461, 8.913116455078125]}, {"step": 700, "epoch": 1, "train_loss": 9.071636199951172, "cos_r_negg": [0.003752725198864937, -0.09578967094421387, -0.757716953754425], "cos_innovation_negg": [-0.056596096605062485, 0.05297227203845978, -0.4917873740196228], "cos_apical_negg": [0.003752725198864937, -0.09578967094421387, -0.757716953754425], "cos_Ac_negg": [-0.05659409984946251, 0.05297340080142021, -0.4918074607849121], "r_norm": [9.147961616516113, 9.273195266723633, 9.56739616394043]}, {"step": 800, "epoch": 1, "train_loss": 18.320663452148438, "cos_r_negg": [0.0030521051958203316, -0.16566374897956848, -0.8253233432769775], "cos_innovation_negg": [-0.10365613549947739, -0.008194144815206528, -0.7189931273460388], "cos_apical_negg": [0.0030521051958203316, -0.16566374897956848, -0.8253233432769775], "cos_Ac_negg": [-0.10364942252635956, -0.00812077522277832, -0.7190485000610352], "r_norm": [9.07063102722168, 9.368144989013672, 9.738855361938477]}, {"step": 900, "epoch": 1, "train_loss": 10.396879196166992, "cos_r_negg": [-0.033398933708667755, 0.07935893535614014, -0.3063790202140808], "cos_innovation_negg": [-0.11821196228265762, -0.17503783106803894, -0.021353432908654213], "cos_apical_negg": [-0.033398933708667755, 0.07935893535614014, -0.3063790202140808], "cos_Ac_negg": [-0.11821117997169495, -0.17491701245307922, -0.021352281793951988], "r_norm": [8.87218189239502, 9.237855911254883, 9.108293533325195]}, {"epoch_end": 1, "step": 938, "test_acc": 0.1009, "test_loss": 10.73256875}, {"step": 1000, "epoch": 2, "train_loss": 9.79171371459961, "cos_r_negg": [-0.02215249091386795, -0.021984010934829712, -0.528407096862793], "cos_innovation_negg": [-0.0030038971453905106, 0.043385595083236694, -0.07335203886032104], "cos_apical_negg": [-0.02215249091386795, -0.021984010934829712, -0.528407096862793], "cos_Ac_negg": [-0.0028996802866458893, 0.04385305568575859, -0.07340101152658463], "r_norm": [8.676126480102539, 8.775712013244629, 8.75094985961914], "ldr": {"dL_sdil": -3.030221939086914, "dL_bp": -3.030221939086914, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 18.787181854248047, "cos_r_negg": [0.03280133008956909, -0.17003311216831207, -0.5385400056838989], "cos_innovation_negg": [0.08627624809741974, 0.14510713517665863, 0.013499096967279911], "cos_apical_negg": [0.03280133008956909, -0.17003311216831207, -0.5385400056838989], "cos_Ac_negg": [0.08627369999885559, 0.1451134979724884, 0.01349706668406725], "r_norm": [8.589005470275879, 8.710916519165039, 8.7020902633667]}, {"step": 1200, "epoch": 2, "train_loss": 13.548810958862305, "cos_r_negg": [0.04559054225683212, -0.023372653871774673, -0.7594349384307861], "cos_innovation_negg": [0.054219406098127365, 0.231641948223114, -0.6000226736068726], "cos_apical_negg": [0.04559054225683212, -0.023372653871774673, -0.7594349384307861], "cos_Ac_negg": [0.054196279495954514, 0.23159898817539215, -0.6000251770019531], "r_norm": [8.217782020568848, 8.14500617980957, 8.383222579956055]}, {"step": 1300, "epoch": 2, "train_loss": 13.265286445617676, "cos_r_negg": [-0.017416872084140778, -0.16834887862205505, -0.796175479888916], "cos_innovation_negg": [-0.08939787745475769, 0.08752074837684631, -0.029462307691574097], "cos_apical_negg": [-0.017416872084140778, -0.16834887862205505, -0.796175479888916], "cos_Ac_negg": [-0.08940528333187103, 0.08751251548528671, -0.029474418610334396], "r_norm": [8.4677734375, 8.719011306762695, 8.723369598388672]}, {"step": 1400, "epoch": 2, "train_loss": 13.85821533203125, "cos_r_negg": [0.024830933660268784, -0.16275572776794434, -0.8117624521255493], "cos_innovation_negg": [0.07545601576566696, 0.2602876126766205, -0.41916966438293457], "cos_apical_negg": [0.024830933660268784, -0.16275572776794434, -0.8117624521255493], "cos_Ac_negg": [0.07547111064195633, 0.2604687809944153, -0.41914069652557373], "r_norm": [8.388879776000977, 8.703527450561523, 8.956075668334961]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.1032, "test_loss": 13.18205888671875}, {"step": 1500, "epoch": 3, "train_loss": 16.52608299255371, "cos_r_negg": [0.0016903132200241089, -0.35976293683052063, -0.42666900157928467], "cos_innovation_negg": [0.10930941998958588, 0.13398471474647522, -0.012624228373169899], "cos_apical_negg": [0.0016903132200241089, -0.35976293683052063, -0.42666900157928467], "cos_Ac_negg": [0.10931478440761566, 0.13423223793506622, -0.012634165585041046], "r_norm": [8.430134773254395, 8.391426086425781, 8.420333862304688], "ldr": {"dL_sdil": -3.7989377975463867, "dL_bp": -3.7989377975463867, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 13.52965259552002, "cos_r_negg": [-0.01540151797235012, -0.49217337369918823, -0.7766873836517334], "cos_innovation_negg": [0.05421750620007515, -0.2627204954624176, -0.7696329951286316], "cos_apical_negg": [-0.01540151797235012, -0.49217337369918823, -0.7766873836517334], "cos_Ac_negg": [0.054326917976140976, -0.2626776695251465, -0.7696387767791748], "r_norm": [8.096517562866211, 7.97392463684082, 8.157990455627441]}, {"step": 1700, "epoch": 3, "train_loss": 6.608334064483643, "cos_r_negg": [0.129010871052742, -0.7731016874313354, -0.8061600923538208], "cos_innovation_negg": [0.12549133598804474, 0.40202921628952026, -0.1931602656841278], "cos_apical_negg": [0.129010871052742, -0.7731016874313354, -0.8061600923538208], "cos_Ac_negg": [0.12555310130119324, 0.40204447507858276, -0.19313877820968628], "r_norm": [8.103960037231445, 8.30439567565918, 8.47163200378418]}, {"step": 1800, "epoch": 3, "train_loss": 8.77731704711914, "cos_r_negg": [0.07717615365982056, -0.7084784507751465, -0.41327208280563354], "cos_innovation_negg": [0.03682496398687363, -0.879778265953064, 0.40282633900642395], "cos_apical_negg": [0.07717615365982056, -0.7084784507751465, -0.41327208280563354], "cos_Ac_negg": [0.036817919462919235, -0.8797733783721924, 0.4028301537036896], "r_norm": [8.304372787475586, 8.354345321655273, 8.349855422973633]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.101, "test_loss": 17.296003515625}, {"step": 1900, "epoch": 4, "train_loss": 13.71055793762207, "cos_r_negg": [0.02283514477312565, -0.4369387626647949, -0.011072501540184021], "cos_innovation_negg": [0.019103897735476494, 0.04924577474594116, 0.3423866927623749], "cos_apical_negg": [0.02283514477312565, -0.4369387626647949, -0.011072501540184021], "cos_Ac_negg": [0.011351268738508224, 0.049227964133024216, 0.34238699078559875], "r_norm": [8.4775390625, 8.527531623840332, 8.589790344238281]}, {"step": 2000, "epoch": 4, "train_loss": 14.162740707397461, "cos_r_negg": [0.0622573085129261, -0.4696023166179657, -0.27911436557769775], "cos_innovation_negg": [-0.05237836763262749, 0.10155412554740906, 0.41249918937683105], "cos_apical_negg": [0.0622573085129261, -0.4696023166179657, -0.27911436557769775], "cos_Ac_negg": [-0.05236773192882538, 0.10154934227466583, 0.41249728202819824], "r_norm": [8.707714080810547, 8.95594596862793, 8.930455207824707], "ldr": {"dL_sdil": -2.1820993423461914, "dL_bp": -2.1820993423461914, "ratio": 1.0}}, {"step": 2100, "epoch": 4, "train_loss": 9.159101486206055, "cos_r_negg": [0.08681570738554001, -0.6591436862945557, -0.6219706535339355], "cos_innovation_negg": [0.057657063007354736, 0.1889386922121048, 0.7060506343841553], "cos_apical_negg": [0.08681570738554001, -0.6591436862945557, -0.6219706535339355], "cos_Ac_negg": [0.057673484086990356, 0.1889382004737854, 0.7060508131980896], "r_norm": [7.795815467834473, 7.828321933746338, 7.75656270980835]}, {"step": 2200, "epoch": 4, "train_loss": 9.759533882141113, "cos_r_negg": [0.10552773624658585, -0.7131375074386597, -0.3689172863960266], "cos_innovation_negg": [-0.01977466233074665, -0.8498791456222534, 0.2792421579360962], "cos_apical_negg": [0.10552773624658585, -0.7131375074386597, -0.3689172863960266], "cos_Ac_negg": [-0.0192576814442873, -0.8495709300041199, 0.27925771474838257], "r_norm": [8.280223846435547, 8.289653778076172, 8.329326629638672]}, {"step": 2300, "epoch": 4, "train_loss": 13.822879791259766, "cos_r_negg": [-0.021266097202897072, -0.6090442538261414, -0.7526376247406006], "cos_innovation_negg": [-0.09191683679819107, 0.013873760588467121, -0.260162353515625], "cos_apical_negg": [-0.021266097202897072, -0.6090442538261414, -0.7526376247406006], "cos_Ac_negg": [-0.09350850433111191, 0.0138530433177948, -0.2601560652256012], "r_norm": [7.853228569030762, 7.725193500518799, 7.814385414123535]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.1135, "test_loss": 15.1349541015625}, {"step": 2400, "epoch": 5, "train_loss": 14.615473747253418, "cos_r_negg": [0.07831874489784241, -0.6990715265274048, -0.45180797576904297], "cos_innovation_negg": [-0.12615223228931427, -0.7499451637268066, 0.30808117985725403], "cos_apical_negg": [0.07831874489784241, -0.6990715265274048, -0.45180797576904297], "cos_Ac_negg": [-0.12614785134792328, -0.7499489784240723, 0.30803513526916504], "r_norm": [8.30342960357666, 8.310445785522461, 8.324758529663086]}, {"step": 2500, "epoch": 5, "train_loss": 7.857935428619385, "cos_r_negg": [0.006961760111153126, 0.21831902861595154, 0.372443288564682], "cos_innovation_negg": [0.010970145463943481, -0.2657153010368347, 0.7116644382476807], "cos_apical_negg": [0.006961760111153126, 0.21831902861595154, 0.372443288564682], "cos_Ac_negg": [0.010974257253110409, -0.2657144367694855, 0.7116653919219971], "r_norm": [8.188138008117676, 8.250946044921875, 8.235197067260742], "ldr": {"dL_sdil": -2.702273368835449, "dL_bp": -2.702273368835449, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 10.229158401489258, "cos_r_negg": [0.047549303621053696, -0.7436808347702026, -0.7110826969146729], "cos_innovation_negg": [-0.10384619981050491, 0.5361222624778748, 0.9698151350021362], "cos_apical_negg": [0.047549303621053696, -0.7436808347702026, -0.7110826969146729], "cos_Ac_negg": [-0.10381660610437393, 0.5361204743385315, 0.9698151350021362], "r_norm": [7.641112327575684, 7.617372512817383, 7.538032531738281]}, {"step": 2700, "epoch": 5, "train_loss": 10.034473419189453, "cos_r_negg": [0.09947732836008072, -0.47585612535476685, -0.03333841264247894], "cos_innovation_negg": [-0.11932677030563354, -0.47150933742523193, 0.7656372785568237], "cos_apical_negg": [0.09947732836008072, -0.47585612535476685, -0.03333841264247894], "cos_Ac_negg": [-0.11924172192811966, -0.4715133607387543, 0.7656365633010864], "r_norm": [8.500308990478516, 8.486753463745117, 8.438323974609375]}, {"step": 2800, "epoch": 5, "train_loss": 8.933666229248047, "cos_r_negg": [0.15354762971401215, -0.03531172126531601, 0.5545244812965393], "cos_innovation_negg": [-0.5570493340492249, -0.31220728158950806, 0.8725370764732361], "cos_apical_negg": [0.15354762971401215, -0.03531172126531601, 0.5545244812965393], "cos_Ac_negg": [-0.5569923520088196, -0.31220948696136475, 0.8725372552871704], "r_norm": [8.191865921020508, 8.185162544250488, 8.222009658813477]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.101, "test_loss": 11.925540234375}, {"step": 2900, "epoch": 6, "train_loss": 8.85418701171875, "cos_r_negg": [-0.00027699136990122497, -0.3002762794494629, 0.35465484857559204], "cos_innovation_negg": [-0.33792492747306824, -0.2030065357685089, 0.299433171749115], "cos_apical_negg": [-0.00027699136990122497, -0.3002762794494629, 0.35465484857559204], "cos_Ac_negg": [-0.338133305311203, -0.2029743492603302, 0.2994375228881836], "r_norm": [8.757638931274414, 8.748701095581055, 8.737950325012207]}, {"step": 3000, "epoch": 6, "train_loss": 9.085477828979492, "cos_r_negg": [0.0003545284562278539, -0.19787263870239258, 0.2468349188566208], "cos_innovation_negg": [-0.1575498878955841, -0.10182784497737885, 0.5882030725479126], "cos_apical_negg": [0.0003545284562278539, -0.19787263870239258, 0.2468349188566208], "cos_Ac_negg": [-0.1576513946056366, -0.10182878375053406, 0.5882030725479126], "r_norm": [8.184025764465332, 8.201793670654297, 8.232870101928711], "ldr": {"dL_sdil": -1.5550522804260254, "dL_bp": -1.5550522804260254, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 15.59103775024414, "cos_r_negg": [-0.24347621202468872, -0.6986595392227173, -0.7824122905731201], "cos_innovation_negg": [-0.3762613534927368, -0.22264257073402405, 0.2640334963798523], "cos_apical_negg": [-0.24347621202468872, -0.6986595392227173, -0.7824122905731201], "cos_Ac_negg": [-0.37618231773376465, -0.22264410555362701, 0.2640328109264374], "r_norm": [8.808782577514648, 8.876242637634277, 8.996013641357422]}, {"step": 3200, "epoch": 6, "train_loss": 7.946864604949951, "cos_r_negg": [-0.1519935131072998, -0.6731467843055725, -0.6288221478462219], "cos_innovation_negg": [-0.24942992627620697, 0.06545795500278473, 0.8332939147949219], "cos_apical_negg": [-0.1519935131072998, -0.6731467843055725, -0.6288221478462219], "cos_Ac_negg": [-0.2498195320367813, 0.06545589864253998, 0.8332937955856323], "r_norm": [8.278830528259277, 8.269832611083984, 8.24264907836914]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.0982, "test_loss": 10.31922021484375}, {"step": 3300, "epoch": 7, "train_loss": 10.728700637817383, "cos_r_negg": [-0.09429904073476791, -0.5135974884033203, -0.41543519496917725], "cos_innovation_negg": [-0.04769972339272499, 0.28234195709228516, 0.36192452907562256], "cos_apical_negg": [-0.09429904073476791, -0.5135974884033203, -0.41543519496917725], "cos_Ac_negg": [-0.04772472381591797, 0.2823414206504822, 0.36192411184310913], "r_norm": [8.627836227416992, 8.710657119750977, 8.804195404052734]}, {"step": 3400, "epoch": 7, "train_loss": 9.546553611755371, "cos_r_negg": [-0.26452872157096863, -0.6543369889259338, -0.5946747064590454], "cos_innovation_negg": [-0.11176694184541702, -0.5381847620010376, 0.589140772819519], "cos_apical_negg": [-0.26452872157096863, -0.6543369889259338, -0.5946747064590454], "cos_Ac_negg": [-0.11247380822896957, -0.5381823778152466, 0.5891375541687012], "r_norm": [8.391143798828125, 8.402189254760742, 8.422823905944824]}, {"step": 3500, "epoch": 7, "train_loss": 10.661148071289062, "cos_r_negg": [-0.45748278498649597, -0.6625232696533203, -0.7758820056915283], "cos_innovation_negg": [-0.2399936467409134, -0.3756147027015686, 0.5112652778625488], "cos_apical_negg": [-0.45748278498649597, -0.6625232696533203, -0.7758820056915283], "cos_Ac_negg": [-0.24028655886650085, -0.37560683488845825, 0.5112648010253906], "r_norm": [8.728376388549805, 8.781932830810547, 8.897253036499023], "ldr": {"dL_sdil": -5.067248821258545, "dL_bp": -5.067248821258545, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 10.910030364990234, "cos_r_negg": [-0.37766072154045105, -0.6576496362686157, -0.6290469765663147], "cos_innovation_negg": [0.018465014174580574, -0.6224530935287476, 0.564693808555603], "cos_apical_negg": [-0.37766072154045105, -0.6576496362686157, -0.6290469765663147], "cos_Ac_negg": [0.02235589362680912, -0.6211690902709961, 0.5646800994873047], "r_norm": [8.363950729370117, 8.37521743774414, 8.417901992797852]}, {"step": 3700, "epoch": 7, "train_loss": 10.535388946533203, "cos_r_negg": [-0.07848601788282394, -0.3434031009674072, -0.38453900814056396], "cos_innovation_negg": [-0.06591659039258957, -0.10192447900772095, 0.45864224433898926], "cos_apical_negg": [-0.07848601788282394, -0.3434031009674072, -0.38453900814056396], "cos_Ac_negg": [-0.0658923089504242, -0.10192491114139557, 0.45864182710647583], "r_norm": [8.332313537597656, 8.332870483398438, 8.325515747070312]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.0982, "test_loss": 16.46620048828125}, {"step": 3800, "epoch": 8, "train_loss": 12.146233558654785, "cos_r_negg": [0.10037415474653244, -0.3377155065536499, -0.007768858224153519], "cos_innovation_negg": [-0.06970730423927307, -0.05333553999662399, 0.5943477749824524], "cos_apical_negg": [0.10037415474653244, -0.3377155065536499, -0.007768858224153519], "cos_Ac_negg": [-0.06973165273666382, -0.05333586409687996, 0.5943472385406494], "r_norm": [8.44619369506836, 8.45177936553955, 8.435260772705078]}, {"step": 3900, "epoch": 8, "train_loss": 12.78567886352539, "cos_r_negg": [-0.4338034689426422, -0.7267639636993408, -0.46289920806884766], "cos_innovation_negg": [0.08336546272039413, -0.13912208378314972, 0.22395376861095428], "cos_apical_negg": [-0.4338034689426422, -0.7267639636993408, -0.46289920806884766], "cos_Ac_negg": [0.08316042274236679, -0.13912124931812286, 0.2239537388086319], "r_norm": [8.301812171936035, 8.270620346069336, 8.347918510437012]}, {"step": 4000, "epoch": 8, "train_loss": 15.174201965332031, "cos_r_negg": [-0.43875402212142944, -0.5560394525527954, -0.6890380382537842], "cos_innovation_negg": [0.4795888066291809, 0.40679773688316345, 0.9682408571243286], "cos_apical_negg": [-0.43875402212142944, -0.5560394525527954, -0.6890380382537842], "cos_Ac_negg": [0.47958746552467346, 0.4067992568016052, 0.9682407975196838], "r_norm": [7.726510524749756, 7.742265701293945, 7.748006343841553], "ldr": {"dL_sdil": -3.7791881561279297, "dL_bp": -3.7791881561279297, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 13.521244049072266, "cos_r_negg": [-0.46494027972221375, -0.6719787120819092, -0.7713680863380432], "cos_innovation_negg": [-0.12176678329706192, -0.28579163551330566, 0.5953699350357056], "cos_apical_negg": [-0.46494027972221375, -0.6719787120819092, -0.7713680863380432], "cos_Ac_negg": [-0.1374925673007965, -0.28530508279800415, 0.5953674912452698], "r_norm": [8.522907257080078, 8.574390411376953, 8.657585144042969]}, {"step": 4200, "epoch": 8, "train_loss": 10.511873245239258, "cos_r_negg": [-0.41155362129211426, -0.46927693486213684, -0.5352820754051208], "cos_innovation_negg": [0.3414989411830902, 0.23229101300239563, 0.7319220304489136], "cos_apical_negg": [-0.41155362129211426, -0.46927693486213684, -0.5352820754051208], "cos_Ac_negg": [0.3415070176124573, 0.23229113221168518, 0.7319220900535583], "r_norm": [7.979081153869629, 7.99635124206543, 7.97769832611084]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.101, "test_loss": 7.894158349609375}, {"step": 4300, "epoch": 9, "train_loss": 8.573269844055176, "cos_r_negg": [-0.4129348695278168, -0.5198704600334167, -0.7027358412742615], "cos_innovation_negg": [-0.4407871961593628, -0.4746224284172058, -0.0009515304118394852], "cos_apical_negg": [-0.4129348695278168, -0.5198704600334167, -0.7027358412742615], "cos_Ac_negg": [-0.44081103801727295, -0.4746224284172058, -0.0009512752294540405], "r_norm": [9.184676170349121, 9.178796768188477, 9.342164039611816]}, {"step": 4400, "epoch": 9, "train_loss": 12.810416221618652, "cos_r_negg": [-0.37855127453804016, -0.5385587811470032, -0.6518173813819885], "cos_innovation_negg": [0.3347189724445343, -0.07439474761486053, 0.9725263118743896], "cos_apical_negg": [-0.37855127453804016, -0.5385587811470032, -0.6518173813819885], "cos_Ac_negg": [0.33469468355178833, -0.07439534366130829, 0.9725263118743896], "r_norm": [8.13730525970459, 8.145676612854004, 8.171100616455078]}, {"step": 4500, "epoch": 9, "train_loss": 10.75411605834961, "cos_r_negg": [0.09120289981365204, -0.38350942730903625, 0.21219921112060547], "cos_innovation_negg": [-0.06338482350111008, 0.07874622941017151, 0.802170991897583], "cos_apical_negg": [0.09120289981365204, -0.38350942730903625, 0.21219921112060547], "cos_Ac_negg": [-0.06338611245155334, 0.07874633371829987, 0.802170991897583], "r_norm": [7.848869323730469, 7.863533973693848, 7.855559349060059], "ldr": {"dL_sdil": -1.8616857528686523, "dL_bp": -1.8616857528686523, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 12.62272834777832, "cos_r_negg": [-0.20304955542087555, -0.6077125668525696, -0.45141518115997314], "cos_innovation_negg": [0.26313459873199463, 0.43562859296798706, 0.6285489201545715], "cos_apical_negg": [-0.20304955542087555, -0.6077125668525696, -0.45141518115997314], "cos_Ac_negg": [0.2631355822086334, 0.43562841415405273, 0.6285489201545715], "r_norm": [7.835481643676758, 7.823282718658447, 7.850198745727539]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.101, "test_loss": 9.423077734375}, {"step": 4700, "epoch": 10, "train_loss": 9.529285430908203, "cos_r_negg": [-0.2656046450138092, -0.671891450881958, -0.6966431140899658], "cos_innovation_negg": [-0.3897179067134857, -0.43567654490470886, 0.7408061623573303], "cos_apical_negg": [-0.2656046450138092, -0.671891450881958, -0.6966431140899658], "cos_Ac_negg": [-0.389731228351593, -0.435676634311676, 0.7408060431480408], "r_norm": [8.641467094421387, 8.64744758605957, 8.714578628540039]}, {"step": 4800, "epoch": 10, "train_loss": 9.602079391479492, "cos_r_negg": [-0.1788291484117508, -0.4288063645362854, -0.31672224402427673], "cos_innovation_negg": [-0.2472972571849823, -0.08655233681201935, 0.3156526982784271], "cos_apical_negg": [-0.1788291484117508, -0.4288063645362854, -0.31672224402427673], "cos_Ac_negg": [-0.24761028587818146, -0.08655165880918503, 0.31565287709236145], "r_norm": [8.557926177978516, 8.588537216186523, 8.649588584899902]}, {"step": 4900, "epoch": 10, "train_loss": 9.333513259887695, "cos_r_negg": [-0.3142012357711792, -0.32894641160964966, -0.5847151279449463], "cos_innovation_negg": [-0.17482060194015503, -0.3071085810661316, 0.5790101885795593], "cos_apical_negg": [-0.3142012357711792, -0.32894641160964966, -0.5847151279449463], "cos_Ac_negg": [-0.17483629286289215, -0.30710864067077637, 0.5790103673934937], "r_norm": [8.824895858764648, 8.833813667297363, 8.915650367736816]}, {"step": 5000, "epoch": 10, "train_loss": 9.437149047851562, "cos_r_negg": [-0.3762127161026001, -0.6445143818855286, -0.7274518609046936], "cos_innovation_negg": [0.4068872034549713, 0.4608134329319, 0.9582438468933105], "cos_apical_negg": [-0.3762127161026001, -0.6445143818855286, -0.7274518609046936], "cos_Ac_negg": [0.4069158136844635, 0.46081334352493286, 0.9582438468933105], "r_norm": [7.98460054397583, 8.009082794189453, 8.047650337219238], "ldr": {"dL_sdil": -2.5101585388183594, "dL_bp": -2.5101585388183594, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 6.1884965896606445, "cos_r_negg": [0.2170650213956833, -0.31876760721206665, -0.0014270832762122154], "cos_innovation_negg": [0.019596539437770844, 0.1470748335123062, 0.6131136417388916], "cos_apical_negg": [0.2170650213956833, -0.31876760721206665, -0.0014270832762122154], "cos_Ac_negg": [0.019468337297439575, 0.1470748335123062, 0.6131137609481812], "r_norm": [8.393791198730469, 8.409490585327148, 8.420076370239258]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.0974, "test_loss": 5.94672236328125}, {"step": 5200, "epoch": 11, "train_loss": 10.347148895263672, "cos_r_negg": [-0.4082636833190918, -0.5161019563674927, -0.7200716733932495], "cos_innovation_negg": [0.22181791067123413, -0.0838456004858017, 0.8572148084640503], "cos_apical_negg": [-0.4082636833190918, -0.5161019563674927, -0.7200716733932495], "cos_Ac_negg": [0.22181324660778046, -0.08384549617767334, 0.8572148084640503], "r_norm": [8.438192367553711, 8.464106559753418, 8.512771606445312]}, {"step": 5300, "epoch": 11, "train_loss": 10.766751289367676, "cos_r_negg": [-0.01570809818804264, -0.3303442597389221, -0.5328904390335083], "cos_innovation_negg": [0.0020136493258178234, -0.09931869059801102, 0.6461442708969116], "cos_apical_negg": [-0.01570809818804264, -0.3303442597389221, -0.5328904390335083], "cos_Ac_negg": [0.0019255706574767828, -0.09931871294975281, 0.6461442708969116], "r_norm": [8.203714370727539, 8.240403175354004, 8.291685104370117]}, {"step": 5400, "epoch": 11, "train_loss": 14.207481384277344, "cos_r_negg": [0.0736018717288971, -0.369031697511673, -0.5387130379676819], "cos_innovation_negg": [0.096673883497715, -0.15685124695301056, 0.37274450063705444], "cos_apical_negg": [0.0736018717288971, -0.369031697511673, -0.5387130379676819], "cos_Ac_negg": [0.0966576561331749, -0.1568520963191986, 0.37274399399757385], "r_norm": [8.43894100189209, 8.468416213989258, 8.547445297241211]}, {"step": 5500, "epoch": 11, "train_loss": 6.16036319732666, "cos_r_negg": [0.31251290440559387, -0.015114255249500275, 0.4244810938835144], "cos_innovation_negg": [-0.05591452121734619, -0.03208724781870842, 0.9253660440444946], "cos_apical_negg": [0.31251290440559387, -0.015114255249500275, 0.4244810938835144], "cos_Ac_negg": [-0.055903200060129166, -0.0320875383913517, 0.9253660440444946], "r_norm": [8.392220497131348, 8.40433406829834, 8.430112838745117], "ldr": {"dL_sdil": -1.5814452171325684, "dL_bp": -1.5814452171325684, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 16.291099548339844, "cos_r_negg": [-0.44796445965766907, -0.5537458658218384, -0.3435918986797333], "cos_innovation_negg": [0.2804306745529175, 0.2043680101633072, 0.19154095649719238], "cos_apical_negg": [-0.44796445965766907, -0.5537458658218384, -0.3435918986797333], "cos_Ac_negg": [0.280397891998291, 0.2043679654598236, 0.19154092669487], "r_norm": [8.222122192382812, 8.233985900878906, 8.235305786132812]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.1135, "test_loss": 11.86322451171875}, {"step": 5700, "epoch": 12, "train_loss": 16.78882598876953, "cos_r_negg": [-0.2476017028093338, -0.5030303597450256, -0.6732052564620972], "cos_innovation_negg": [0.08175352960824966, -0.13145005702972412, 0.7364891171455383], "cos_apical_negg": [-0.2476017028093338, -0.5030303597450256, -0.6732052564620972], "cos_Ac_negg": [0.08174634724855423, -0.13144907355308533, 0.7364890575408936], "r_norm": [8.413293838500977, 8.419957160949707, 8.488061904907227]}, {"step": 5800, "epoch": 12, "train_loss": 15.03429889678955, "cos_r_negg": [-0.3591378331184387, -0.6063828468322754, -0.7384556531906128], "cos_innovation_negg": [0.17521454393863678, 0.4509395658969879, 0.982961893081665], "cos_apical_negg": [-0.3591378331184387, -0.6063828468322754, -0.7384556531906128], "cos_Ac_negg": [0.17548583447933197, 0.45093923807144165, 0.982961893081665], "r_norm": [7.920623779296875, 7.945095062255859, 7.976653575897217]}, {"step": 5900, "epoch": 12, "train_loss": 7.8121232986450195, "cos_r_negg": [-0.28654351830482483, -0.5653884410858154, -0.6129470467567444], "cos_innovation_negg": [0.0925280824303627, 0.4283583164215088, 0.6947625279426575], "cos_apical_negg": [-0.28654351830482483, -0.5653884410858154, -0.6129470467567444], "cos_Ac_negg": [0.09234333783388138, 0.42835816740989685, 0.6947625279426575], "r_norm": [8.129790306091309, 8.14554500579834, 8.163032531738281]}, {"step": 6000, "epoch": 12, "train_loss": 9.103167533874512, "cos_r_negg": [0.060810863971710205, -0.21191316843032837, -0.3813336491584778], "cos_innovation_negg": [0.27643728256225586, 0.01874524913728237, 0.4911237359046936], "cos_apical_negg": [0.060810863971710205, -0.21191316843032837, -0.3813336491584778], "cos_Ac_negg": [0.2581687569618225, 0.0186927393078804, 0.49112939834594727], "r_norm": [8.685422897338867, 8.708581924438477, 8.861541748046875], "ldr": {"dL_sdil": -5.562556266784668, "dL_bp": -5.562556266784668, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.1028, "test_loss": 9.17819130859375}, {"step": 6100, "epoch": 13, "train_loss": 7.973690509796143, "cos_r_negg": [-0.05185166746377945, -0.41801899671554565, -0.5955506563186646], "cos_innovation_negg": [0.21301838755607605, 0.3214159309864044, 0.892964780330658], "cos_apical_negg": [-0.05185166746377945, -0.41801899671554565, -0.5955506563186646], "cos_Ac_negg": [0.21304172277450562, 0.32141605019569397, 0.8929648399353027], "r_norm": [7.935324668884277, 7.950346946716309, 7.931529998779297]}, {"step": 6200, "epoch": 13, "train_loss": 5.477077484130859, "cos_r_negg": [-0.3294876217842102, -0.3743704855442047, -0.5970391035079956], "cos_innovation_negg": [0.26121431589126587, 0.22947219014167786, 0.8951684832572937], "cos_apical_negg": [-0.3294876217842102, -0.3743704855442047, -0.5970391035079956], "cos_Ac_negg": [0.2612909972667694, 0.22947360575199127, 0.8951684236526489], "r_norm": [8.0214204788208, 8.03666877746582, 8.099931716918945]}, {"step": 6300, "epoch": 13, "train_loss": 9.624935150146484, "cos_r_negg": [-0.39674413204193115, -0.5298537015914917, -0.6483625173568726], "cos_innovation_negg": [-0.25022029876708984, -0.41722628474235535, 0.6040381193161011], "cos_apical_negg": [-0.39674413204193115, -0.5298537015914917, -0.6483625173568726], "cos_Ac_negg": [-0.250224769115448, -0.41722622513771057, 0.6040381193161011], "r_norm": [8.693485260009766, 8.702049255371094, 8.782125473022461]}, {"step": 6400, "epoch": 13, "train_loss": 13.130498886108398, "cos_r_negg": [-0.3625592291355133, -0.6538761854171753, -0.6579976677894592], "cos_innovation_negg": [-0.3745867609977722, -0.4442564845085144, 0.7031722664833069], "cos_apical_negg": [-0.3625592291355133, -0.6538761854171753, -0.6579976677894592], "cos_Ac_negg": [-0.37459123134613037, -0.44425660371780396, 0.7031713724136353], "r_norm": [8.71908950805664, 8.73489761352539, 8.811705589294434]}, {"step": 6500, "epoch": 13, "train_loss": 13.624479293823242, "cos_r_negg": [0.31674566864967346, -0.15249276161193848, 0.30055463314056396], "cos_innovation_negg": [-0.22038350999355316, -0.0383467897772789, 0.8990703821182251], "cos_apical_negg": [0.31674566864967346, -0.15249276161193848, 0.30055463314056396], "cos_Ac_negg": [-0.22038047015666962, -0.03834675997495651, 0.8990703225135803], "r_norm": [7.634829998016357, 7.650902271270752, 7.616977691650391], "ldr": {"dL_sdil": -2.8467493057250977, "dL_bp": -2.8467493057250977, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.0974, "test_loss": 12.99880166015625}, {"step": 6600, "epoch": 14, "train_loss": 10.941816329956055, "cos_r_negg": [0.3845733404159546, 0.25478658080101013, 0.6958084106445312], "cos_innovation_negg": [-0.36162856221199036, -0.3567724823951721, 0.8712005615234375], "cos_apical_negg": [0.3845733404159546, 0.25478658080101013, 0.6958084106445312], "cos_Ac_negg": [-0.3615977168083191, -0.35677260160446167, 0.871200680732727], "r_norm": [7.598811149597168, 7.616349220275879, 7.582597732543945]}, {"step": 6700, "epoch": 14, "train_loss": 14.647473335266113, "cos_r_negg": [-0.13037341833114624, -0.4976232647895813, -0.5071238875389099], "cos_innovation_negg": [0.13961094617843628, 0.09501032531261444, 0.6536878943443298], "cos_apical_negg": [-0.13037341833114624, -0.4976232647895813, -0.5071238875389099], "cos_Ac_negg": [0.13962233066558838, 0.09501034021377563, 0.6536878943443298], "r_norm": [8.156566619873047, 8.17742919921875, 8.2016019821167]}, {"step": 6800, "epoch": 14, "train_loss": 9.072355270385742, "cos_r_negg": [0.27423563599586487, 0.04300399869680405, 0.0011499146930873394], "cos_innovation_negg": [-0.3992229402065277, -0.41233015060424805, 0.7035161852836609], "cos_apical_negg": [0.27423563599586487, 0.04300399869680405, 0.0011499146930873394], "cos_Ac_negg": [-0.39921072125434875, -0.4123303294181824, 0.7035162448883057], "r_norm": [8.303342819213867, 8.315505027770996, 8.33897876739502]}, {"step": 6900, "epoch": 14, "train_loss": 15.945040702819824, "cos_r_negg": [-0.24756598472595215, -0.4225154221057892, -0.5549700260162354], "cos_innovation_negg": [-0.44082409143447876, -0.6881178617477417, 0.5757876038551331], "cos_apical_negg": [-0.24756598472595215, -0.4225154221057892, -0.5549700260162354], "cos_Ac_negg": [-0.44082918763160706, -0.6881176829338074, 0.5757876038551331], "r_norm": [8.821377754211426, 8.830106735229492, 8.891897201538086]}, {"step": 7000, "epoch": 14, "train_loss": 19.636981964111328, "cos_r_negg": [-0.24000513553619385, -0.6569687128067017, -0.7477624416351318], "cos_innovation_negg": [-0.4084286391735077, -0.7700883150100708, 0.15814308822155], "cos_apical_negg": [-0.24000513553619385, -0.6569687128067017, -0.7477624416351318], "cos_Ac_negg": [-0.40842878818511963, -0.770088791847229, 0.1581439971923828], "r_norm": [9.132789611816406, 9.151155471801758, 9.241266250610352], "ldr": {"dL_sdil": -2.704376220703125, "dL_bp": -2.704376220703125, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1135, "test_loss": 14.7281623046875}], "final": {"test_acc": 0.1135, "test_loss": 14.7281623046875, "wall_s": 24.486724138259888, "cos_r_negg": [-0.20963400602340698, -0.6111788153648376, -0.5194591283798218], "cos_innovation_negg": [-0.12342631071805954, 0.15727558732032776, 0.8628395795822144], "cos_apical_negg": [-0.20963400602340698, -0.6111788153648376, -0.5194591283798218], "cos_Ac_negg": [-0.12344752252101898, 0.15727408230304718, 0.8628397583961487]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_raw_s1.json b/results/nuis_ctrl_v1_rho0p5_raw_s1.json new file mode 100644 index 0000000..ee88da7 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_raw_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_raw_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.019747942686080933, -0.012253424152731895, -0.010189061984419823], "cos_innovation_negg": [0.032370083034038544, -0.011818937957286835, -0.006588317919522524], "cos_apical_negg": [0.019747942686080933, -0.012253424152731895, -0.010189061984419823], "cos_Ac_negg": [0.03236745670437813, -0.011820998974144459, -0.0065889982506632805], "r_norm": [8.022297859191895, 6.936640739440918, 6.413236618041992], "ldr": {"dL_sdil": 0.03699207305908203, "dL_bp": -0.16228103637695312, "ratio": -0.22795068287064243}}, {"step": 100, "epoch": 0, "train_loss": 10.255548477172852, "cos_r_negg": [0.019994664937257767, 0.1572040170431137, -0.39453125], "cos_innovation_negg": [0.026966765522956848, 0.23288485407829285, 0.1953125], "cos_apical_negg": [0.019994664937257767, 0.1572040170431137, -0.39453125], "cos_Ac_negg": [0.026951849460601807, 0.23288394510746002, 0.1953125], "r_norm": [10.35006046295166, 9.845701217651367, 9.498064994812012]}, {"step": 200, "epoch": 0, "train_loss": 8.509317398071289, "cos_r_negg": [-0.028591888025403023, 0.01346166804432869, -0.8125], "cos_innovation_negg": [-0.04961170628666878, -0.08859357237815857, 0.625], "cos_apical_negg": [-0.028591888025403023, 0.01346166804432869, -0.8125], "cos_Ac_negg": [-0.04960859194397926, -0.08857496082782745, 0.625], "r_norm": [10.754764556884766, 10.037720680236816, 10.064632415771484]}, {"step": 300, "epoch": 0, "train_loss": 17.83721160888672, "cos_r_negg": [-0.057352762669324875, -0.0851992815732956, -0.6171875], "cos_innovation_negg": [-0.041011743247509, -0.05717262253165245, -0.02734375], "cos_apical_negg": [-0.057352762669324875, -0.0851992815732956, -0.6171875], "cos_Ac_negg": [-0.04092578589916229, -0.057143256068229675, -0.02734375], "r_norm": [9.938236236572266, 9.55727767944336, 9.466873168945312]}, {"step": 400, "epoch": 0, "train_loss": 7.2290825843811035, "cos_r_negg": [0.11731478571891785, 0.27728092670440674, -0.58984375], "cos_innovation_negg": [0.15003573894500732, 0.16362930834293365, 0.0], "cos_apical_negg": [0.11731478571891785, 0.27728092670440674, -0.58984375], "cos_Ac_negg": [0.1500292718410492, 0.1636141985654831, 0.0], "r_norm": [9.497688293457031, 9.26835823059082, 9.029476165771484]}, {"epoch_end": 0, "step": 469, "test_acc": 0.1028, "test_loss": 11.4579041015625}, {"step": 500, "epoch": 1, "train_loss": 17.226133346557617, "cos_r_negg": [0.040775030851364136, 0.10044033080339432, -0.8046875], "cos_innovation_negg": [0.041798170655965805, 0.018371954560279846, 0.02734375], "cos_apical_negg": [0.040775030851364136, 0.10044033080339432, -0.8046875], "cos_Ac_negg": [0.041797056794166565, 0.018360260874032974, 0.02734375], "r_norm": [9.502269744873047, 9.298660278320312, 9.243734359741211], "ldr": {"dL_sdil": -2.046304702758789, "dL_bp": -2.046304702758789, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 11.26502513885498, "cos_r_negg": [-0.012791586108505726, 0.040445100516080856, -0.8046875], "cos_innovation_negg": [0.017647143453359604, 0.04069416970014572, 0.4296875], "cos_apical_negg": [-0.012791586108505726, 0.040445100516080856, -0.8046875], "cos_Ac_negg": [0.017636168748140335, 0.04069536551833153, 0.4296875], "r_norm": [9.236584663391113, 8.872383117675781, 8.91727066040039]}, {"step": 700, "epoch": 1, "train_loss": 11.11388874053955, "cos_r_negg": [0.013096985407173634, -0.15340036153793335, -0.484375], "cos_innovation_negg": [-0.040903255343437195, -0.2035406231880188, 0.4296875], "cos_apical_negg": [0.013096985407173634, -0.15340036153793335, -0.484375], "cos_Ac_negg": [-0.04156612977385521, -0.20494085550308228, 0.4296875], "r_norm": [9.124551773071289, 9.2476806640625, 9.073410034179688]}, {"step": 800, "epoch": 1, "train_loss": 12.390909194946289, "cos_r_negg": [-0.015279165469110012, 0.05406123772263527, -0.640625], "cos_innovation_negg": [-0.03402719646692276, -0.05897021293640137, -0.6484375], "cos_apical_negg": [-0.015279165469110012, 0.05406123772263527, -0.640625], "cos_Ac_negg": [-0.03403740003705025, -0.058980755507946014, -0.6484375], "r_norm": [8.950582504272461, 9.133920669555664, 9.43021011352539]}, {"step": 900, "epoch": 1, "train_loss": 11.088262557983398, "cos_r_negg": [-0.011971227824687958, -0.16905423998832703, -0.3125], "cos_innovation_negg": [-0.05211924761533737, -0.2989688515663147, 0.6015625], "cos_apical_negg": [-0.011971227824687958, -0.16905423998832703, -0.3125], "cos_Ac_negg": [-0.05179830640554428, -0.29844844341278076, 0.6015625], "r_norm": [8.590007781982422, 8.724785804748535, 8.576166152954102]}, {"epoch_end": 1, "step": 938, "test_acc": 0.0958, "test_loss": 10.70472607421875}, {"step": 1000, "epoch": 2, "train_loss": 7.3060622215271, "cos_r_negg": [0.010516166687011719, -0.11391425132751465, -0.8046875], "cos_innovation_negg": [-0.04206956923007965, 0.28496024012565613, 0.4296875], "cos_apical_negg": [0.010516166687011719, -0.11391425132751465, -0.8046875], "cos_Ac_negg": [-0.042079176753759384, 0.28496259450912476, 0.4296875], "r_norm": [8.405755043029785, 8.252345085144043, 8.325518608093262], "ldr": {"dL_sdil": -1.3452386856079102, "dL_bp": -1.3452386856079102, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 14.472942352294922, "cos_r_negg": [-0.011405833065509796, -0.06569328904151917, -0.6171875], "cos_innovation_negg": [0.02072197198867798, 0.23664258420467377, 0.19140625], "cos_apical_negg": [-0.011405833065509796, -0.06569328904151917, -0.6171875], "cos_Ac_negg": [0.020735356956720352, 0.23664231598377228, 0.19140625], "r_norm": [8.573768615722656, 8.777698516845703, 8.76460075378418]}, {"step": 1200, "epoch": 2, "train_loss": 12.044473648071289, "cos_r_negg": [0.014382035471498966, -0.22828882932662964, -0.796875], "cos_innovation_negg": [-0.05162845551967621, 0.24645277857780457, 1.0], "cos_apical_negg": [0.014382035471498966, -0.22828882932662964, -0.796875], "cos_Ac_negg": [-0.05164097994565964, 0.2464318722486496, 1.0], "r_norm": [8.376583099365234, 8.457967758178711, 8.215885162353516]}, {"step": 1300, "epoch": 2, "train_loss": 6.5063629150390625, "cos_r_negg": [0.017980705946683884, -0.09746164083480835, -0.58984375], "cos_innovation_negg": [0.024920618161559105, 0.1227433979511261, -0.00390625], "cos_apical_negg": [0.017980705946683884, -0.09746164083480835, -0.58984375], "cos_Ac_negg": [0.024907678365707397, 0.12276691198348999, -0.00390625], "r_norm": [7.911869049072266, 7.881860733032227, 7.70210599899292]}, {"step": 1400, "epoch": 2, "train_loss": 9.817485809326172, "cos_r_negg": [0.030312037095427513, -0.30478209257125854, -0.8046875], "cos_innovation_negg": [0.054487232118844986, 0.20171639323234558, 0.4296875], "cos_apical_negg": [0.030312037095427513, -0.30478209257125854, -0.8046875], "cos_Ac_negg": [0.05447912961244583, 0.2017064243555069, 0.4296875], "r_norm": [8.463221549987793, 8.475896835327148, 8.521780014038086]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.1032, "test_loss": 11.7731794921875}, {"step": 1500, "epoch": 3, "train_loss": 11.53721809387207, "cos_r_negg": [0.05731525644659996, -0.038483504205942154, -0.8046875], "cos_innovation_negg": [0.15988948941230774, -0.16430561244487762, -0.484375], "cos_apical_negg": [0.05731525644659996, -0.038483504205942154, -0.8046875], "cos_Ac_negg": [0.15984931588172913, -0.16432219743728638, -0.484375], "r_norm": [8.08842945098877, 8.42395305633545, 8.705155372619629], "ldr": {"dL_sdil": -2.464749336242676, "dL_bp": -2.464749336242676, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 8.154747009277344, "cos_r_negg": [-0.027677316218614578, -0.08400880545377731, -0.6171875], "cos_innovation_negg": [0.04914071038365364, -0.031338125467300415, 0.0234375], "cos_apical_negg": [-0.027677316218614578, -0.08400880545377731, -0.6171875], "cos_Ac_negg": [0.04915933310985565, -0.03130488842725754, 0.0234375], "r_norm": [8.72436237335205, 8.844265937805176, 8.89121150970459]}, {"step": 1700, "epoch": 3, "train_loss": 6.9549784660339355, "cos_r_negg": [-0.03542960062623024, 0.022684037685394287, -0.2421875], "cos_innovation_negg": [-0.06088436022400856, 0.08918435871601105, 0.5859375], "cos_apical_negg": [-0.03542960062623024, 0.022684037685394287, -0.2421875], "cos_Ac_negg": [-0.06088074669241905, 0.08920293301343918, 0.5859375], "r_norm": [8.389348030090332, 8.436502456665039, 8.428319931030273]}, {"step": 1800, "epoch": 3, "train_loss": 6.131024360656738, "cos_r_negg": [0.035309989005327225, -0.2508002817630768, -0.8046875], "cos_innovation_negg": [-0.0997670516371727, 0.292773962020874, 0.2109375], "cos_apical_negg": [0.035309989005327225, -0.2508002817630768, -0.8046875], "cos_Ac_negg": [-0.099770687520504, 0.2927319407463074, 0.2109375], "r_norm": [8.066953659057617, 7.979948043823242, 7.825244903564453]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.0958, "test_loss": 15.22708388671875}, {"step": 1900, "epoch": 4, "train_loss": 11.77390193939209, "cos_r_negg": [0.08595188707113266, -0.5049331188201904, -0.6171875], "cos_innovation_negg": [-0.07709004729986191, 0.2698267698287964, 0.21484375], "cos_apical_negg": [0.08595188707113266, -0.5049331188201904, -0.6171875], "cos_Ac_negg": [-0.07708196341991425, 0.2698175311088562, 0.21484375], "r_norm": [8.575183868408203, 8.621679306030273, 8.65350341796875]}, {"step": 2000, "epoch": 4, "train_loss": 11.215351104736328, "cos_r_negg": [-0.009597906842827797, -0.3489310145378113, -0.421875], "cos_innovation_negg": [-0.02166447415947914, 0.18506285548210144, -0.16015625], "cos_apical_negg": [-0.009597906842827797, -0.3489310145378113, -0.421875], "cos_Ac_negg": [-0.02163964882493019, 0.18505842983722687, -0.16015625], "r_norm": [8.188566207885742, 8.312118530273438, 8.16801643371582], "ldr": {"dL_sdil": -1.9362831115722656, "dL_bp": -1.9362821578979492, "ratio": 1.0000004925285877}}, {"step": 2100, "epoch": 4, "train_loss": 14.727140426635742, "cos_r_negg": [0.014038543216884136, -0.674130916595459, -0.8046875], "cos_innovation_negg": [-0.024987127631902695, -0.42598649859428406, 0.0], "cos_apical_negg": [0.014038543216884136, -0.674130916595459, -0.8046875], "cos_Ac_negg": [-0.024976588785648346, -0.42599087953567505, 0.0], "r_norm": [8.830476760864258, 8.85820198059082, 8.99573802947998]}, {"step": 2200, "epoch": 4, "train_loss": 10.289848327636719, "cos_r_negg": [0.029824919998645782, -0.3603938817977905, -0.4296875], "cos_innovation_negg": [0.0957798957824707, 0.0723714679479599, 0.375], "cos_apical_negg": [0.029824919998645782, -0.3603938817977905, -0.4296875], "cos_Ac_negg": [0.09578461199998856, 0.07236636430025101, 0.375], "r_norm": [8.682836532592773, 8.686776161193848, 8.668912887573242]}, {"step": 2300, "epoch": 4, "train_loss": 9.162446975708008, "cos_r_negg": [0.09142471849918365, -0.6800583004951477, -0.8046875], "cos_innovation_negg": [-0.1525205671787262, -0.09147711098194122, 0.0234375], "cos_apical_negg": [0.09142471849918365, -0.6800583004951477, -0.8046875], "cos_Ac_negg": [-0.15252111852169037, -0.09148325026035309, 0.0234375], "r_norm": [8.597091674804688, 8.609399795532227, 8.657405853271484]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.0958, "test_loss": 11.5237857421875}, {"step": 2400, "epoch": 5, "train_loss": 14.908965110778809, "cos_r_negg": [0.2208012491464615, -0.6925382614135742, -0.8125], "cos_innovation_negg": [-0.016925405710935593, 0.1201857477426529, -0.1953125], "cos_apical_negg": [0.2208012491464615, -0.6925382614135742, -0.8125], "cos_Ac_negg": [-0.010413708165287971, 0.12075091898441315, -0.1953125], "r_norm": [8.76136589050293, 8.790179252624512, 8.832708358764648]}, {"step": 2500, "epoch": 5, "train_loss": 9.113646507263184, "cos_r_negg": [0.05305565521121025, -0.6685764193534851, -0.78515625], "cos_innovation_negg": [-0.006513811647891998, -0.4660874009132385, -0.1953125], "cos_apical_negg": [0.05305565521121025, -0.6685764193534851, -0.78515625], "cos_Ac_negg": [-0.006514234934002161, -0.4660630524158478, -0.1953125], "r_norm": [8.754865646362305, 8.771283149719238, 8.807271957397461], "ldr": {"dL_sdil": -3.0341272354125977, "dL_bp": -3.0341272354125977, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 11.880722045898438, "cos_r_negg": [-0.10047242045402527, -0.325664222240448, -0.3984375], "cos_innovation_negg": [-0.17108644545078278, 0.05257571488618851, 0.19140625], "cos_apical_negg": [-0.10047242045402527, -0.325664222240448, -0.3984375], "cos_Ac_negg": [-0.17110908031463623, 0.052568379789590836, 0.19140625], "r_norm": [8.301441192626953, 8.415926933288574, 8.310450553894043]}, {"step": 2700, "epoch": 5, "train_loss": 11.036749839782715, "cos_r_negg": [-0.20094235241413116, -0.6670376062393188, -0.8046875], "cos_innovation_negg": [-0.10651130229234695, -0.23025165498256683, 0.4296875], "cos_apical_negg": [-0.20094235241413116, -0.6670376062393188, -0.8046875], "cos_Ac_negg": [-0.10646158456802368, -0.23025259375572205, 0.4296875], "r_norm": [8.485367774963379, 8.47578239440918, 8.530881881713867]}, {"step": 2800, "epoch": 5, "train_loss": 23.670425415039062, "cos_r_negg": [-0.13499264419078827, -0.7049131393432617, -0.8359375], "cos_innovation_negg": [0.08350823819637299, -0.4116736650466919, -1.0], "cos_apical_negg": [-0.13499264419078827, -0.7049131393432617, -0.8359375], "cos_Ac_negg": [0.08351808786392212, -0.41167405247688293, -1.0], "r_norm": [8.97557258605957, 9.089457511901855, 9.230287551879883]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.0982, "test_loss": 11.89017373046875}, {"step": 2900, "epoch": 6, "train_loss": 10.981536865234375, "cos_r_negg": [-0.18273533880710602, -0.6601389646530151, -0.8046875], "cos_innovation_negg": [0.2024480402469635, -0.30576205253601074, 0.19140625], "cos_apical_negg": [-0.18273533880710602, -0.6601389646530151, -0.8046875], "cos_Ac_negg": [0.20248305797576904, -0.3057602047920227, 0.19140625], "r_norm": [8.712766647338867, 8.71328353881836, 8.782217979431152]}, {"step": 3000, "epoch": 6, "train_loss": 12.230125427246094, "cos_r_negg": [-0.1647941619157791, -0.49704235792160034, -0.6015625], "cos_innovation_negg": [-0.08325158804655075, 0.590111494064331, 0.8046875], "cos_apical_negg": [-0.1647941619157791, -0.49704235792160034, -0.6015625], "cos_Ac_negg": [-0.08324957638978958, 0.5901111364364624, 0.8046875], "r_norm": [7.905643463134766, 7.888843536376953, 7.838493347167969], "ldr": {"dL_sdil": -1.1818304061889648, "dL_bp": -1.1818304061889648, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 12.053131103515625, "cos_r_negg": [-0.2135394811630249, -0.6514860391616821, -0.78515625], "cos_innovation_negg": [0.25970882177352905, -0.3043336272239685, -0.00390625], "cos_apical_negg": [-0.2135394811630249, -0.6514860391616821, -0.78515625], "cos_Ac_negg": [0.25969114899635315, -0.30433493852615356, -0.00390625], "r_norm": [8.471572875976562, 8.472350120544434, 8.493823051452637]}, {"step": 3200, "epoch": 6, "train_loss": 11.591326713562012, "cos_r_negg": [-0.2027834802865982, -0.6619418859481812, -0.8046875], "cos_innovation_negg": [0.13849501311779022, -0.3411552906036377, -0.37109375], "cos_apical_negg": [-0.2027834802865982, -0.6619418859481812, -0.8046875], "cos_Ac_negg": [0.13849510252475739, -0.34115615487098694, -0.37109375], "r_norm": [8.946978569030762, 9.021909713745117, 9.154298782348633]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.1135, "test_loss": 7.05854375}, {"step": 3300, "epoch": 7, "train_loss": 17.59749984741211, "cos_r_negg": [-0.08525792509317398, -0.35279908776283264, -0.42578125], "cos_innovation_negg": [0.08196399360895157, -0.3393060266971588, -0.58984375], "cos_apical_negg": [-0.08525792509317398, -0.35279908776283264, -0.42578125], "cos_Ac_negg": [0.08196558058261871, -0.33930644392967224, -0.58984375], "r_norm": [9.134340286254883, 9.198554039001465, 9.27498722076416]}, {"step": 3400, "epoch": 7, "train_loss": 14.165465354919434, "cos_r_negg": [-0.0696418508887291, -0.16134686768054962, -0.1953125], "cos_innovation_negg": [0.36436980962753296, 0.09845741093158722, 0.17578125], "cos_apical_negg": [-0.0696418508887291, -0.16134686768054962, -0.1953125], "cos_Ac_negg": [0.3667339086532593, 0.09845790266990662, 0.17578125], "r_norm": [8.55362606048584, 8.533271789550781, 8.539839744567871]}, {"step": 3500, "epoch": 7, "train_loss": 16.618972778320312, "cos_r_negg": [-0.21431967616081238, -0.6796319484710693, -0.828125], "cos_innovation_negg": [0.047853875905275345, 0.24010559916496277, 0.6015625], "cos_apical_negg": [-0.21431967616081238, -0.6796319484710693, -0.828125], "cos_Ac_negg": [0.04785699397325516, 0.24010488390922546, 0.6015625], "r_norm": [8.068330764770508, 8.11501407623291, 8.141497611999512], "ldr": {"dL_sdil": -2.8917312622070312, "dL_bp": -2.8917312622070312, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 17.49654769897461, "cos_r_negg": [-0.05472403019666672, -0.6517399549484253, -0.796875], "cos_innovation_negg": [0.2943275272846222, 0.1049397736787796, 0.41796875], "cos_apical_negg": [-0.05472403019666672, -0.6517399549484253, -0.796875], "cos_Ac_negg": [0.29432404041290283, 0.10493916273117065, 0.41796875], "r_norm": [8.525372505187988, 8.524673461914062, 8.515222549438477]}, {"step": 3700, "epoch": 7, "train_loss": 10.19696044921875, "cos_r_negg": [-0.22893820703029633, -0.6602572798728943, -0.8046875], "cos_innovation_negg": [0.2143981009721756, 0.029962781816720963, 0.19140625], "cos_apical_negg": [-0.22893820703029633, -0.6602572798728943, -0.8046875], "cos_Ac_negg": [0.21438570320606232, 0.02996237389743328, 0.19140625], "r_norm": [8.477952003479004, 8.456353187561035, 8.47607135772705]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.098, "test_loss": 9.13295830078125}, {"step": 3800, "epoch": 8, "train_loss": 9.953214645385742, "cos_r_negg": [-0.19311043620109558, -0.6523042321205139, -0.8046875], "cos_innovation_negg": [0.2931424677371979, 0.2158634513616562, 0.41015625], "cos_apical_negg": [-0.19311043620109558, -0.6523042321205139, -0.8046875], "cos_Ac_negg": [0.29322537779808044, 0.21586306393146515, 0.41015625], "r_norm": [8.249199867248535, 8.287460327148438, 8.300777435302734]}, {"step": 3900, "epoch": 8, "train_loss": 6.829085350036621, "cos_r_negg": [-0.20397111773490906, -0.6477262377738953, -0.8046875], "cos_innovation_negg": [0.17422093451023102, 0.44347500801086426, 0.625], "cos_apical_negg": [-0.20397111773490906, -0.6477262377738953, -0.8046875], "cos_Ac_negg": [0.1742263287305832, 0.44347482919692993, 0.625], "r_norm": [8.033866882324219, 8.039278030395508, 8.032636642456055]}, {"step": 4000, "epoch": 8, "train_loss": 10.8768310546875, "cos_r_negg": [-0.13105210661888123, -0.25949379801750183, -0.31640625], "cos_innovation_negg": [-0.10799551010131836, -0.32377392053604126, -0.421875], "cos_apical_negg": [-0.13105210661888123, -0.25949379801750183, -0.31640625], "cos_Ac_negg": [-0.10805001109838486, -0.3237740099430084, -0.421875], "r_norm": [8.77724838256836, 8.795995712280273, 8.798884391784668], "ldr": {"dL_sdil": -1.1976318359375, "dL_bp": -1.1976318359375, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 13.615104675292969, "cos_r_negg": [0.04135633632540703, 0.04593779146671295, 0.05859375], "cos_innovation_negg": [0.2123039811849594, 0.08738474547863007, 0.359375], "cos_apical_negg": [0.04135633632540703, 0.04593779146671295, 0.05859375], "cos_Ac_negg": [0.2123110294342041, 0.08738459646701813, 0.359375], "r_norm": [8.383914947509766, 8.420074462890625, 8.391923904418945]}, {"step": 4200, "epoch": 8, "train_loss": 18.92833709716797, "cos_r_negg": [-0.13118402659893036, -0.4789707064628601, -0.58984375], "cos_innovation_negg": [0.2658495604991913, 0.11149245500564575, 0.46875], "cos_apical_negg": [-0.13118402659893036, -0.4789707064628601, -0.58984375], "cos_Ac_negg": [0.2658456563949585, 0.11149241775274277, 0.46875], "r_norm": [8.485389709472656, 8.5098237991333, 8.531970977783203]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.0982, "test_loss": 13.37454501953125}, {"step": 4300, "epoch": 9, "train_loss": 10.261911392211914, "cos_r_negg": [-0.09637166559696198, -0.5956960320472717, -0.73046875], "cos_innovation_negg": [-0.3425491750240326, -0.6807563900947571, -0.8359375], "cos_apical_negg": [-0.09637166559696198, -0.5956960320472717, -0.73046875], "cos_Ac_negg": [-0.3426416516304016, -0.6807563900947571, -0.8359375], "r_norm": [8.899179458618164, 8.92036247253418, 8.970678329467773]}, {"step": 4400, "epoch": 9, "train_loss": 10.811222076416016, "cos_r_negg": [-0.1516168862581253, -0.658687949180603, -0.8125], "cos_innovation_negg": [0.07316669821739197, -0.07895853370428085, -0.16015625], "cos_apical_negg": [-0.1516168862581253, -0.658687949180603, -0.8125], "cos_Ac_negg": [0.07483233511447906, -0.07895766943693161, -0.16015625], "r_norm": [8.702716827392578, 8.70215129852295, 8.762594223022461]}, {"step": 4500, "epoch": 9, "train_loss": 11.859151840209961, "cos_r_negg": [-0.09459742903709412, -0.47759270668029785, -0.58984375], "cos_innovation_negg": [0.2598411738872528, 0.034248627722263336, 0.46875], "cos_apical_negg": [-0.09459742903709412, -0.47759270668029785, -0.58984375], "cos_Ac_negg": [0.25983455777168274, 0.03424864262342453, 0.46875], "r_norm": [8.425926208496094, 8.45444107055664, 8.458234786987305], "ldr": {"dL_sdil": -1.6856966018676758, "dL_bp": -1.6856956481933594, "ratio": 1.0000005657452562}}, {"step": 4600, "epoch": 9, "train_loss": 13.60978889465332, "cos_r_negg": [-0.262395441532135, -0.6500300168991089, -0.8046875], "cos_innovation_negg": [0.1120535135269165, -0.09465036541223526, 0.625], "cos_apical_negg": [-0.262395441532135, -0.6500300168991089, -0.8046875], "cos_Ac_negg": [0.11534196883440018, -0.09464994072914124, 0.625], "r_norm": [8.468623161315918, 8.479522705078125, 8.519412994384766]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.1028, "test_loss": 19.0580978515625}, {"step": 4700, "epoch": 10, "train_loss": 13.967798233032227, "cos_r_negg": [-0.19663439691066742, -0.6369585990905762, -0.78515625], "cos_innovation_negg": [0.018648996949195862, -0.1480116993188858, -0.34765625], "cos_apical_negg": [-0.19663439691066742, -0.6369585990905762, -0.78515625], "cos_Ac_negg": [0.018578849732875824, -0.14801178872585297, -0.34765625], "r_norm": [8.440733909606934, 8.4483642578125, 8.463715553283691]}, {"step": 4800, "epoch": 10, "train_loss": 6.129626274108887, "cos_r_negg": [-0.10255283862352371, -0.6352357864379883, -0.78515625], "cos_innovation_negg": [-0.10318326950073242, -0.16379831731319427, -0.34765625], "cos_apical_negg": [-0.10255283862352371, -0.6352357864379883, -0.78515625], "cos_Ac_negg": [-0.10324013233184814, -0.1637982428073883, -0.34765625], "r_norm": [8.470785140991211, 8.488651275634766, 8.482808113098145]}, {"step": 4900, "epoch": 10, "train_loss": 12.707818984985352, "cos_r_negg": [0.040788255631923676, -0.004775119945406914, -0.00390625], "cos_innovation_negg": [0.289998322725296, 0.4808981120586395, 0.6484375], "cos_apical_negg": [0.040788255631923676, -0.004775119945406914, -0.00390625], "cos_Ac_negg": [0.28999438881874084, 0.4808982014656067, 0.6484375], "r_norm": [8.638195037841797, 8.648427963256836, 8.758851051330566]}, {"step": 5000, "epoch": 10, "train_loss": 7.799116134643555, "cos_r_negg": [-0.03289207071065903, 0.07600315660238266, 0.09375], "cos_innovation_negg": [-0.03519286960363388, 0.356702983379364, 0.41796875], "cos_apical_negg": [-0.03289207071065903, 0.07600315660238266, 0.09375], "cos_Ac_negg": [-0.035194121301174164, 0.356702983379364, 0.41796875], "r_norm": [8.377689361572266, 8.400264739990234, 8.440694808959961], "ldr": {"dL_sdil": -1.9702091217041016, "dL_bp": -1.9702091217041016, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 10.47219467163086, "cos_r_negg": [-0.03785387799143791, -0.48272615671157837, -0.6015625], "cos_innovation_negg": [0.07259231805801392, 0.48528632521629333, 0.6328125], "cos_apical_negg": [-0.03785387799143791, -0.48272615671157837, -0.6015625], "cos_Ac_negg": [0.07262501865625381, 0.4852863848209381, 0.6328125], "r_norm": [7.70461368560791, 7.7034382820129395, 7.708110332489014]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.0892, "test_loss": 12.00000849609375}, {"step": 5200, "epoch": 11, "train_loss": 10.71782112121582, "cos_r_negg": [-0.18562965095043182, -0.5159285068511963, -0.640625], "cos_innovation_negg": [-0.03629561886191368, -0.4750930964946747, -0.31640625], "cos_apical_negg": [-0.18562965095043182, -0.5159285068511963, -0.640625], "cos_Ac_negg": [-0.03630782663822174, -0.47509312629699707, -0.31640625], "r_norm": [9.172325134277344, 9.21832275390625, 9.317131996154785]}, {"step": 5300, "epoch": 11, "train_loss": 5.943276405334473, "cos_r_negg": [0.061552613973617554, 0.041206296533346176, 0.05078125], "cos_innovation_negg": [0.08826838433742523, -0.12186316400766373, -0.1640625], "cos_apical_negg": [0.061552613973617554, 0.041206296533346176, 0.05078125], "cos_Ac_negg": [0.08823709189891815, -0.12186320126056671, -0.1640625], "r_norm": [8.823055267333984, 8.838237762451172, 8.881866455078125]}, {"step": 5400, "epoch": 11, "train_loss": 10.271692276000977, "cos_r_negg": [-0.15554723143577576, -0.4915660619735718, -0.6171875], "cos_innovation_negg": [0.0002655188145581633, 0.017148923128843307, 0.19140625], "cos_apical_negg": [-0.15554723143577576, -0.4915660619735718, -0.6171875], "cos_Ac_negg": [0.00026533339405432343, 0.017149074003100395, 0.19140625], "r_norm": [7.972602844238281, 7.987299919128418, 7.990858554840088]}, {"step": 5500, "epoch": 11, "train_loss": 12.208606719970703, "cos_r_negg": [-0.19998127222061157, -0.6461871862411499, -0.8046875], "cos_innovation_negg": [-0.08988816291093826, 0.005642070434987545, 0.40625], "cos_apical_negg": [-0.19998127222061157, -0.6461871862411499, -0.8046875], "cos_Ac_negg": [-0.08987265080213547, 0.005642108619213104, 0.40625], "r_norm": [8.203372955322266, 8.207939147949219, 8.234451293945312], "ldr": {"dL_sdil": -1.4868803024291992, "dL_bp": -1.4868803024291992, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 8.259819030761719, "cos_r_negg": [0.05633822828531265, -0.33073586225509644, -0.4140625], "cos_innovation_negg": [0.11186914145946503, 0.27397483587265015, 0.39453125], "cos_apical_negg": [0.05633822828531265, -0.33073586225509644, -0.4140625], "cos_Ac_negg": [0.11187991499900818, 0.2739748954772949, 0.39453125], "r_norm": [8.12523365020752, 8.145190238952637, 8.183408737182617]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.098, "test_loss": 13.60080322265625}, {"step": 5700, "epoch": 12, "train_loss": 7.852395057678223, "cos_r_negg": [-0.28524982929229736, -0.6453471183776855, -0.8046875], "cos_innovation_negg": [0.05096225440502167, -0.10975302010774612, 0.78125], "cos_apical_negg": [-0.28524982929229736, -0.6453471183776855, -0.8046875], "cos_Ac_negg": [0.051130689680576324, -0.1097530722618103, 0.78125], "r_norm": [8.443368911743164, 8.461749076843262, 8.447426795959473]}, {"step": 5800, "epoch": 12, "train_loss": 12.04050064086914, "cos_r_negg": [-0.1931564211845398, -0.3389608561992645, -0.421875], "cos_innovation_negg": [0.3688060939311981, 0.4867904782295227, 0.640625], "cos_apical_negg": [-0.1931564211845398, -0.3389608561992645, -0.421875], "cos_Ac_negg": [0.3688081204891205, 0.4867904484272003, 0.640625], "r_norm": [7.668402671813965, 7.678630352020264, 7.639505386352539]}, {"step": 5900, "epoch": 12, "train_loss": 9.473217010498047, "cos_r_negg": [-0.20913226902484894, -0.6424360871315002, -0.8046875], "cos_innovation_negg": [-0.008895385079085827, -0.022748205810785294, 0.21875], "cos_apical_negg": [-0.20913226902484894, -0.6424360871315002, -0.8046875], "cos_Ac_negg": [-0.008909438736736774, -0.022748293355107307, 0.21875], "r_norm": [8.045585632324219, 8.056964874267578, 8.067422866821289]}, {"step": 6000, "epoch": 12, "train_loss": 4.6162109375, "cos_r_negg": [-0.048817843198776245, -0.3077565133571625, -0.38671875], "cos_innovation_negg": [0.03845749422907829, -0.09262891858816147, 0.265625], "cos_apical_negg": [-0.048817843198776245, -0.3077565133571625, -0.38671875], "cos_Ac_negg": [0.03863661363720894, -0.09262891113758087, 0.265625], "r_norm": [8.418392181396484, 8.431730270385742, 8.438087463378906], "ldr": {"dL_sdil": -1.201819896697998, "dL_bp": -1.201819896697998, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.1028, "test_loss": 8.93193388671875}, {"step": 6100, "epoch": 13, "train_loss": 6.096339225769043, "cos_r_negg": [0.12168270349502563, -0.1704285591840744, -0.21484375], "cos_innovation_negg": [-0.10440283268690109, 0.012353229336440563, 0.37109375], "cos_apical_negg": [0.12168270349502563, -0.1704285591840744, -0.21484375], "cos_Ac_negg": [-0.10440690815448761, 0.012353229336440563, 0.37109375], "r_norm": [7.819033622741699, 7.832849979400635, 7.832224369049072]}, {"step": 6200, "epoch": 13, "train_loss": 7.847175121307373, "cos_r_negg": [-0.048842836171388626, -0.5008080005645752, -0.6328125], "cos_innovation_negg": [0.06866491585969925, 0.4265682101249695, 0.8046875], "cos_apical_negg": [-0.048842836171388626, -0.5008080005645752, -0.6328125], "cos_Ac_negg": [0.06857999414205551, 0.42656785249710083, 0.8046875], "r_norm": [7.716916561126709, 7.72566032409668, 7.794162273406982]}, {"step": 6300, "epoch": 13, "train_loss": 7.977858066558838, "cos_r_negg": [-0.09518624097108841, -0.46747344732284546, -0.5859375], "cos_innovation_negg": [0.11895081400871277, 0.5588080883026123, 0.8046875], "cos_apical_negg": [-0.09518624097108841, -0.46747344732284546, -0.5859375], "cos_Ac_negg": [0.11893948167562485, 0.5588085651397705, 0.8046875], "r_norm": [7.622480869293213, 7.630342483520508, 7.614863395690918]}, {"step": 6400, "epoch": 13, "train_loss": 9.694928169250488, "cos_r_negg": [0.07838189601898193, -0.34082505106925964, -0.4296875], "cos_innovation_negg": [-0.19109903275966644, -0.07497815042734146, 0.19140625], "cos_apical_negg": [0.07838189601898193, -0.34082505106925964, -0.4296875], "cos_Ac_negg": [-0.19108383357524872, -0.07497777789831161, 0.19140625], "r_norm": [8.213350296020508, 8.22939682006836, 8.26217269897461]}, {"step": 6500, "epoch": 13, "train_loss": 9.161271095275879, "cos_r_negg": [-0.09720490127801895, -0.17380550503730774, -0.21875], "cos_innovation_negg": [-0.24518650770187378, -0.010658462531864643, 0.40234375], "cos_apical_negg": [-0.09720490127801895, -0.17380550503730774, -0.21875], "cos_Ac_negg": [-0.24520382285118103, -0.010658405721187592, 0.40234375], "r_norm": [8.145859718322754, 8.16114330291748, 8.184585571289062], "ldr": {"dL_sdil": -1.7441749572753906, "dL_bp": -1.7441740036010742, "ratio": 1.00000054677705}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.101, "test_loss": 11.3324763671875}, {"step": 6600, "epoch": 14, "train_loss": 10.590892791748047, "cos_r_negg": [-0.023335391655564308, 0.014607099816203117, 0.01953125], "cos_innovation_negg": [-0.2865261137485504, -0.3393670618534088, -0.1640625], "cos_apical_negg": [-0.023335391655564308, 0.014607099816203117, 0.01953125], "cos_Ac_negg": [-0.28653809428215027, -0.3393670916557312, -0.1640625], "r_norm": [8.343790054321289, 8.360283851623535, 8.364086151123047]}, {"step": 6700, "epoch": 14, "train_loss": 7.228377342224121, "cos_r_negg": [0.11366799473762512, -0.210951030254364, -0.265625], "cos_innovation_negg": [-0.3458082675933838, -0.3371458947658539, 0.16796875], "cos_apical_negg": [0.11366799473762512, -0.210951030254364, -0.265625], "cos_Ac_negg": [-0.3459179401397705, -0.337146133184433, 0.16796875], "r_norm": [8.159287452697754, 8.17740249633789, 8.223154067993164]}, {"step": 6800, "epoch": 14, "train_loss": 10.455390930175781, "cos_r_negg": [-0.12113658338785172, -0.635749340057373, -0.796875], "cos_innovation_negg": [-0.01943564973771572, 0.00035860203206539154, 0.40625], "cos_apical_negg": [-0.12113658338785172, -0.635749340057373, -0.796875], "cos_Ac_negg": [-0.019462957978248596, 0.00035974010825157166, 0.40625], "r_norm": [8.254014015197754, 8.264790534973145, 8.324295043945312]}, {"step": 6900, "epoch": 14, "train_loss": 11.511268615722656, "cos_r_negg": [-0.2255789190530777, -0.6680182218551636, -0.8359375], "cos_innovation_negg": [-0.24502550065517426, -0.37816905975341797, -0.56640625], "cos_apical_negg": [-0.2255789190530777, -0.6680182218551636, -0.8359375], "cos_Ac_negg": [-0.24504362046718597, -0.37816905975341797, -0.56640625], "r_norm": [8.650300025939941, 8.67970085144043, 8.728560447692871]}, {"step": 7000, "epoch": 14, "train_loss": 13.167366027832031, "cos_r_negg": [-0.19883595407009125, -0.6421783566474915, -0.8046875], "cos_innovation_negg": [-0.2787978947162628, -0.12518706917762756, 0.16015625], "cos_apical_negg": [-0.19883595407009125, -0.6421783566474915, -0.8046875], "cos_Ac_negg": [-0.2788105010986328, -0.1251869797706604, 0.16015625], "r_norm": [8.168951988220215, 8.182437896728516, 8.196771621704102], "ldr": {"dL_sdil": -1.5362911224365234, "dL_bp": -1.5362911224365234, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1009, "test_loss": 8.2447443359375}], "final": {"test_acc": 0.1009, "test_loss": 8.2447443359375, "wall_s": 22.726298093795776, "cos_r_negg": [-0.19490432739257812, -0.6292647123336792, -0.78515625], "cos_innovation_negg": [-0.1356714516878128, -0.28259608149528503, -0.078125], "cos_apical_negg": [-0.19490432739257812, -0.6292647123336792, -0.78515625], "cos_Ac_negg": [-0.13538646697998047, -0.28259608149528503, -0.078125]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_raw_s2.json b/results/nuis_ctrl_v1_rho0p5_raw_s2.json new file mode 100644 index 0000000..d416383 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_raw_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_raw_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.006069933995604515, 0.028213273733854294, 0.000850403041113168], "cos_innovation_negg": [0.006572579964995384, 0.03945208340883255, 0.0016749617643654346], "cos_apical_negg": [0.006069933995604515, 0.028213273733854294, 0.000850403041113168], "cos_Ac_negg": [0.006574587430804968, 0.03945256024599075, 0.0016756001859903336], "r_norm": [8.09968090057373, 6.984550476074219, 6.564377784729004], "ldr": {"dL_sdil": -0.040816545486450195, "dL_bp": -0.1888291835784912, "ratio": 0.2161559178138577}}, {"step": 100, "epoch": 0, "train_loss": 8.607678413391113, "cos_r_negg": [0.043286263942718506, 0.02338339574635029, 0.01075268816202879], "cos_innovation_negg": [0.07204825431108475, 0.2613758444786072, -0.5655913949012756], "cos_apical_negg": [0.043286263942718506, 0.02338339574635029, 0.01075268816202879], "cos_Ac_negg": [0.07190786302089691, 0.2612081468105316, -0.5655913949012756], "r_norm": [10.865097999572754, 10.794242858886719, 10.00767707824707]}, {"step": 200, "epoch": 0, "train_loss": 14.23398208618164, "cos_r_negg": [-0.05621775984764099, -0.06742869317531586, -0.2560291588306427], "cos_innovation_negg": [-0.037481822073459625, 0.14581120014190674, -0.5063558220863342], "cos_apical_negg": [-0.05621775984764099, -0.06742869317531586, -0.2560291588306427], "cos_Ac_negg": [-0.03706343472003937, 0.1459081619977951, -0.5063564777374268], "r_norm": [10.567609786987305, 9.59535026550293, 9.80006217956543]}, {"step": 300, "epoch": 0, "train_loss": 7.152601718902588, "cos_r_negg": [-0.004294459708034992, -0.036559201776981354, 0.39814814925193787], "cos_innovation_negg": [-0.025412902235984802, 0.18632517755031586, 0.375], "cos_apical_negg": [-0.004294459708034992, -0.036559201776981354, 0.39814814925193787], "cos_Ac_negg": [-0.025513848289847374, 0.18630953133106232, 0.375], "r_norm": [9.94304084777832, 9.509531021118164, 9.850784301757812]}, {"step": 400, "epoch": 0, "train_loss": 11.771880149841309, "cos_r_negg": [0.02575899101793766, 0.20267914235591888, -0.46919432282447815], "cos_innovation_negg": [-0.011691069230437279, 0.26164400577545166, -0.2132701575756073], "cos_apical_negg": [0.02575899101793766, 0.20267914235591888, -0.46919432282447815], "cos_Ac_negg": [-0.011695858091115952, 0.26161447167396545, -0.2132701575756073], "r_norm": [9.711320877075195, 9.536703109741211, 9.406965255737305]}, {"epoch_end": 0, "step": 469, "test_acc": 0.0892, "test_loss": 5.7297685546875}, {"step": 500, "epoch": 1, "train_loss": 7.890752792358398, "cos_r_negg": [-0.00917352270334959, 0.009967482648789883, -0.4682352840900421], "cos_innovation_negg": [-0.04460432752966881, 0.25391969084739685, -0.030588233843445778], "cos_apical_negg": [-0.00917352270334959, 0.009967482648789883, -0.4682352840900421], "cos_Ac_negg": [-0.04462495446205139, 0.2539190351963043, -0.030588233843445778], "r_norm": [9.352081298828125, 9.198244094848633, 9.062174797058105], "ldr": {"dL_sdil": -0.7840461730957031, "dL_bp": -0.7840461730957031, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 8.19209098815918, "cos_r_negg": [-0.025425070896744728, 0.09573765099048615, 0.6352822184562683], "cos_innovation_negg": [-0.02527477592229843, 0.260063111782074, -0.9988667964935303], "cos_apical_negg": [-0.025425070896744728, 0.09573765099048615, 0.6352822184562683], "cos_Ac_negg": [-0.02532678283751011, 0.2600637972354889, -0.9988669157028198], "r_norm": [9.43122673034668, 9.104082107543945, 9.184896469116211]}, {"step": 700, "epoch": 1, "train_loss": 10.612958908081055, "cos_r_negg": [0.011208701878786087, -0.020073596388101578, 0.18309859931468964], "cos_innovation_negg": [0.05172453820705414, 0.2757587432861328, -0.07511737197637558], "cos_apical_negg": [0.011208701878786087, -0.020073596388101578, 0.18309859931468964], "cos_Ac_negg": [0.05172889307141304, 0.2757216989994049, -0.07511737197637558], "r_norm": [9.012006759643555, 8.955952644348145, 9.195959091186523]}, {"step": 800, "epoch": 1, "train_loss": 7.530488967895508, "cos_r_negg": [0.01302462350577116, -0.06212879344820976, -0.4751131534576416], "cos_innovation_negg": [0.02423812448978424, 0.19756846129894257, 0.0], "cos_apical_negg": [0.01302462350577116, -0.06212879344820976, -0.4751131534576416], "cos_Ac_negg": [0.024241534993052483, 0.19757811725139618, 0.0], "r_norm": [8.682754516601562, 8.557674407958984, 8.5389986038208]}, {"step": 900, "epoch": 1, "train_loss": 9.516929626464844, "cos_r_negg": [0.019926870241761208, -0.11089521646499634, -0.5698447823524475], "cos_innovation_negg": [0.04081986844539642, 0.33930930495262146, -0.6585365533828735], "cos_apical_negg": [0.019926870241761208, -0.11089521646499634, -0.5698447823524475], "cos_Ac_negg": [0.04129152372479439, 0.33924928307533264, -0.6585365533828735], "r_norm": [8.787179946899414, 8.46986198425293, 8.557413101196289]}, {"epoch_end": 1, "step": 938, "test_acc": 0.0958, "test_loss": 14.253339453125}, {"step": 1000, "epoch": 2, "train_loss": 11.903938293457031, "cos_r_negg": [-0.01657753251492977, -0.01202379260212183, 0.2232142984867096], "cos_innovation_negg": [-0.16749955713748932, 0.013943457044661045, 0.2008928656578064], "cos_apical_negg": [-0.01657753251492977, -0.01202379260212183, 0.2232142984867096], "cos_Ac_negg": [-0.1675061583518982, 0.01390395499765873, 0.2008928656578064], "r_norm": [8.835855484008789, 9.250875473022461, 8.899043083190918], "ldr": {"dL_sdil": -1.3018312454223633, "dL_bp": -1.3018312454223633, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 7.095750331878662, "cos_r_negg": [0.028792718425393105, 0.016014961525797844, -0.20770877599716187], "cos_innovation_negg": [0.003021157579496503, 0.043102797120809555, -0.010706637986004353], "cos_apical_negg": [0.028792718425393105, 0.016014961525797844, -0.20770877599716187], "cos_Ac_negg": [0.0030201401095837355, 0.04310184344649315, -0.010706637986004353], "r_norm": [8.63536262512207, 8.829824447631836, 8.958860397338867]}, {"step": 1200, "epoch": 2, "train_loss": 17.46408462524414, "cos_r_negg": [-0.023260295391082764, 0.1216074749827385, -0.4345114231109619], "cos_innovation_negg": [-0.018750956282019615, -0.11288260668516159, 0.82536381483078], "cos_apical_negg": [-0.023260295391082764, 0.1216074749827385, -0.4345114231109619], "cos_Ac_negg": [-0.01872233673930168, -0.11265920847654343, 0.82536381483078], "r_norm": [8.729228973388672, 8.940746307373047, 9.051870346069336]}, {"step": 1300, "epoch": 2, "train_loss": 13.53596305847168, "cos_r_negg": [0.025372661650180817, -0.006017977371811867, -0.42578125], "cos_innovation_negg": [-0.07081383466720581, -0.15056994557380676, -0.2421875], "cos_apical_negg": [0.025372661650180817, -0.006017977371811867, -0.42578125], "cos_Ac_negg": [-0.07078513503074646, -0.15052571892738342, -0.2421875], "r_norm": [8.374276161193848, 8.17935562133789, 8.201964378356934]}, {"step": 1400, "epoch": 2, "train_loss": 10.240300178527832, "cos_r_negg": [0.0203335490077734, 0.27930134534835815, 0.5859375], "cos_innovation_negg": [0.11001616716384888, 0.19296517968177795, 0.36328125], "cos_apical_negg": [0.0203335490077734, 0.27930134534835815, 0.5859375], "cos_Ac_negg": [0.10946781188249588, 0.19257411360740662, 0.36328125], "r_norm": [9.051124572753906, 9.292688369750977, 9.421853065490723]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.1028, "test_loss": 12.33814736328125}, {"step": 1500, "epoch": 3, "train_loss": 16.444610595703125, "cos_r_negg": [-0.0047040837816894054, -0.2855634391307831, -0.265625], "cos_innovation_negg": [0.04879166930913925, 0.22736075520515442, 0.34765625], "cos_apical_negg": [-0.0047040837816894054, -0.2855634391307831, -0.265625], "cos_Ac_negg": [0.04879727214574814, 0.22735252976417542, 0.34765625], "r_norm": [8.130634307861328, 8.34067153930664, 8.304515838623047], "ldr": {"dL_sdil": -1.3420801162719727, "dL_bp": -1.3420801162719727, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 9.035846710205078, "cos_r_negg": [-0.003250044770538807, -0.20443597435951233, -0.21875], "cos_innovation_negg": [-0.016233408823609352, 0.14438506960868835, -0.01171875], "cos_apical_negg": [-0.003250044770538807, -0.20443597435951233, -0.21875], "cos_Ac_negg": [-0.016230184584856033, 0.14437103271484375, -0.01171875], "r_norm": [8.45271110534668, 8.598628997802734, 8.630088806152344]}, {"step": 1700, "epoch": 3, "train_loss": 13.682973861694336, "cos_r_negg": [0.008910275995731354, 0.5314902067184448, 0.5859375], "cos_innovation_negg": [-0.04581741616129875, 0.2546497583389282, 0.58203125], "cos_apical_negg": [0.008910275995731354, 0.5314902067184448, 0.5859375], "cos_Ac_negg": [-0.04582258313894272, 0.2546376585960388, 0.58203125], "r_norm": [8.633129119873047, 8.776876449584961, 8.845964431762695]}, {"step": 1800, "epoch": 3, "train_loss": 18.205738067626953, "cos_r_negg": [0.04461752995848656, -0.47234925627708435, -0.484375], "cos_innovation_negg": [0.0671183317899704, -0.16538923978805542, 0.78125], "cos_apical_negg": [0.04461752995848656, -0.47234925627708435, -0.484375], "cos_Ac_negg": [0.06705057621002197, -0.1654069423675537, 0.78125], "r_norm": [8.202865600585938, 8.286920547485352, 8.286673545837402]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.1032, "test_loss": 11.171415625}, {"step": 1900, "epoch": 4, "train_loss": 11.181070327758789, "cos_r_negg": [-0.06173565611243248, 0.09183380007743835, 0.11328125], "cos_innovation_negg": [0.0642334595322609, -0.040925033390522, 0.51171875], "cos_apical_negg": [-0.06173565611243248, 0.09183380007743835, 0.11328125], "cos_Ac_negg": [0.06420228630304337, -0.04095173254609108, 0.51171875], "r_norm": [8.587827682495117, 8.683780670166016, 8.845311164855957]}, {"step": 2000, "epoch": 4, "train_loss": 14.013265609741211, "cos_r_negg": [-0.03492019325494766, 0.10401894152164459, 0.11328125], "cos_innovation_negg": [-0.03786076605319977, -0.02154957689344883, 0.78515625], "cos_apical_negg": [-0.03492019325494766, 0.10401894152164459, 0.11328125], "cos_Ac_negg": [-0.03788123279809952, -0.021539095789194107, 0.78515625], "r_norm": [8.202961921691895, 8.270047187805176, 8.323112487792969], "ldr": {"dL_sdil": -1.8188533782958984, "dL_bp": -1.8188533782958984, "ratio": 1.0}}, {"step": 2100, "epoch": 4, "train_loss": 6.470243453979492, "cos_r_negg": [-0.0008236072608269751, -0.05182565748691559, -0.05078125], "cos_innovation_negg": [0.0933600664138794, -0.1702737808227539, 0.15234375], "cos_apical_negg": [-0.0008236072608269751, -0.05182565748691559, -0.05078125], "cos_Ac_negg": [0.09327870607376099, -0.17038270831108093, 0.15234375], "r_norm": [8.264612197875977, 8.322214126586914, 8.388392448425293]}, {"step": 2200, "epoch": 4, "train_loss": 10.790571212768555, "cos_r_negg": [-0.06804899871349335, -0.0514930784702301, -0.05078125], "cos_innovation_negg": [-0.14114263653755188, -0.04461637884378433, 0.56640625], "cos_apical_negg": [-0.06804899871349335, -0.0514930784702301, -0.05078125], "cos_Ac_negg": [-0.14115355908870697, -0.044624097645282745, 0.56640625], "r_norm": [8.39930534362793, 8.48637580871582, 8.370820999145508]}, {"step": 2300, "epoch": 4, "train_loss": 11.865091323852539, "cos_r_negg": [-0.07438310235738754, -0.05125802755355835, -0.05078125], "cos_innovation_negg": [-0.15275540947914124, 0.1405005156993866, 0.8359375], "cos_apical_negg": [-0.07438310235738754, -0.05125802755355835, -0.05078125], "cos_Ac_negg": [-0.15322351455688477, 0.13474678993225098, 0.8359375], "r_norm": [8.323041915893555, 8.41018295288086, 8.291877746582031]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.0958, "test_loss": 9.8487564453125}, {"step": 2400, "epoch": 5, "train_loss": 9.967241287231445, "cos_r_negg": [0.04543090984225273, -0.10339514911174774, -0.1015625], "cos_innovation_negg": [-0.04555587098002434, -0.23899617791175842, 0.78515625], "cos_apical_negg": [0.04543090984225273, -0.10339514911174774, -0.1015625], "cos_Ac_negg": [-0.04559876769781113, -0.2389882504940033, 0.78515625], "r_norm": [8.396261215209961, 8.439985275268555, 8.51211929321289]}, {"step": 2500, "epoch": 5, "train_loss": 14.84898567199707, "cos_r_negg": [-0.04485081508755684, -0.2646341025829315, -0.26953125], "cos_innovation_negg": [-0.25339028239250183, 0.06275664269924164, 0.12890625], "cos_apical_negg": [-0.04485081508755684, -0.2646341025829315, -0.26953125], "cos_Ac_negg": [-0.2533881366252899, 0.0627472847700119, 0.12890625], "r_norm": [8.10526180267334, 8.128353118896484, 8.154784202575684], "ldr": {"dL_sdil": -2.2104549407958984, "dL_bp": -2.2104549407958984, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 17.929655075073242, "cos_r_negg": [0.10165649652481079, -0.7821450233459473, -0.8125], "cos_innovation_negg": [-0.23728157579898834, -0.541267991065979, 0.23828125], "cos_apical_negg": [0.10165649652481079, -0.7821450233459473, -0.8125], "cos_Ac_negg": [-0.2373037338256836, -0.5412731170654297, 0.23828125], "r_norm": [8.668704986572266, 8.701017379760742, 8.750303268432617]}, {"step": 2700, "epoch": 5, "train_loss": 9.901445388793945, "cos_r_negg": [0.009795840829610825, 0.5564864873886108, 0.5859375], "cos_innovation_negg": [-0.030734334141016006, 0.02718561515212059, 0.796875], "cos_apical_negg": [0.009795840829610825, 0.5564864873886108, 0.5859375], "cos_Ac_negg": [-0.030703900381922722, 0.02718810923397541, 0.796875], "r_norm": [8.112945556640625, 8.154273986816406, 8.163640975952148]}, {"step": 2800, "epoch": 5, "train_loss": 15.594620704650879, "cos_r_negg": [0.12130018323659897, -0.18450616300106049, -0.20703125], "cos_innovation_negg": [0.09369021654129028, 0.09002826362848282, 0.19140625], "cos_apical_negg": [0.12130018323659897, -0.18450616300106049, -0.20703125], "cos_Ac_negg": [0.09368741512298584, 0.0744839608669281, 0.19140625], "r_norm": [8.264737129211426, 8.303056716918945, 8.341985702514648]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.1028, "test_loss": 20.358579296875}, {"step": 2900, "epoch": 6, "train_loss": 16.89208984375, "cos_r_negg": [-0.014575080946087837, -0.2592034637928009, -0.265625], "cos_innovation_negg": [0.0833456739783287, -0.1388159841299057, 0.8125], "cos_apical_negg": [-0.014575080946087837, -0.2592034637928009, -0.265625], "cos_Ac_negg": [0.0833434984087944, -0.13881127536296844, 0.8125], "r_norm": [8.071036338806152, 8.065722465515137, 8.09593677520752]}, {"step": 3000, "epoch": 6, "train_loss": 11.365478515625, "cos_r_negg": [0.047974660992622375, -0.6225115060806274, -0.640625], "cos_innovation_negg": [0.019792458042502403, -0.43774449825286865, 0.4375], "cos_apical_negg": [0.047974660992622375, -0.6225115060806274, -0.640625], "cos_Ac_negg": [0.01979689858853817, -0.43774959444999695, 0.4375], "r_norm": [8.169724464416504, 8.164944648742676, 8.221406936645508], "ldr": {"dL_sdil": -2.4173011779785156, "dL_bp": -2.4173011779785156, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 14.204668045043945, "cos_r_negg": [0.02127210609614849, -0.26237258315086365, -0.26953125], "cos_innovation_negg": [-0.03792629763484001, -0.3400703966617584, 0.78125], "cos_apical_negg": [0.02127210609614849, -0.26237258315086365, -0.26953125], "cos_Ac_negg": [-0.03792750835418701, -0.34007057547569275, 0.78125], "r_norm": [8.430807113647461, 8.42910099029541, 8.496881484985352]}, {"step": 3200, "epoch": 6, "train_loss": 11.091636657714844, "cos_r_negg": [0.015585962682962418, -0.8003131747245789, -0.828125], "cos_innovation_negg": [0.12115953117609024, -0.7471803426742554, 0.78125], "cos_apical_negg": [0.015585962682962418, -0.8003131747245789, -0.828125], "cos_Ac_negg": [0.12116257101297379, -0.7471810579299927, 0.78125], "r_norm": [8.819147109985352, 8.829202651977539, 8.839988708496094]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.0974, "test_loss": 8.17626220703125}, {"step": 3300, "epoch": 7, "train_loss": 11.65943431854248, "cos_r_negg": [-0.07276209443807602, -0.5372828245162964, -0.56640625], "cos_innovation_negg": [0.06662250310182571, 0.558137834072113, 0.828125], "cos_apical_negg": [-0.07276209443807602, -0.5372828245162964, -0.56640625], "cos_Ac_negg": [0.06661158800125122, 0.5581486225128174, 0.828125], "r_norm": [7.814938545227051, 7.757964611053467, 7.753844261169434]}, {"step": 3400, "epoch": 7, "train_loss": 9.130935668945312, "cos_r_negg": [-0.06863047927618027, -0.8006556630134583, -0.828125], "cos_innovation_negg": [0.02796264924108982, -0.7023664712905884, 1.0], "cos_apical_negg": [-0.06863047927618027, -0.8006556630134583, -0.828125], "cos_Ac_negg": [0.027953851968050003, -0.702366828918457, 1.0], "r_norm": [8.880365371704102, 8.896379470825195, 8.904964447021484]}, {"step": 3500, "epoch": 7, "train_loss": 9.979610443115234, "cos_r_negg": [5.192796015762724e-05, -0.7991563081741333, -0.828125], "cos_innovation_negg": [0.03197434917092323, -0.6567003726959229, 0.78125], "cos_apical_negg": [5.192796015762724e-05, -0.7991563081741333, -0.828125], "cos_Ac_negg": [0.031970880925655365, -0.6567010283470154, 0.78125], "r_norm": [8.684354782104492, 8.69792366027832, 8.674968719482422], "ldr": {"dL_sdil": -1.5388154983520508, "dL_bp": -1.5388145446777344, "ratio": 1.0000006197461024}}, {"step": 3600, "epoch": 7, "train_loss": 12.586816787719727, "cos_r_negg": [-0.012873931787908077, 0.2093500941991806, 0.21875], "cos_innovation_negg": [-0.11395280808210373, -0.3456953167915344, 0.8359375], "cos_apical_negg": [-0.012873931787908077, 0.2093500941991806, 0.21875], "cos_Ac_negg": [-0.11394085735082626, -0.34569495916366577, 0.8359375], "r_norm": [7.8183088302612305, 7.841979026794434, 7.790170669555664]}, {"step": 3700, "epoch": 7, "train_loss": 10.789178848266602, "cos_r_negg": [0.013845272362232208, 0.3656242787837982, 0.3828125], "cos_innovation_negg": [-0.21636852622032166, -0.49038785696029663, 0.796875], "cos_apical_negg": [0.013845272362232208, 0.3656242787837982, 0.3828125], "cos_Ac_negg": [-0.2163427770137787, -0.4903874397277832, 0.796875], "r_norm": [8.18099594116211, 8.203760147094727, 8.175332069396973]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.1028, "test_loss": 8.26958056640625}, {"step": 3800, "epoch": 8, "train_loss": 10.51145076751709, "cos_r_negg": [-0.05470547825098038, -0.3788420259952545, -0.39453125], "cos_innovation_negg": [-0.0007461022469215095, -0.27109163999557495, 0.4921875], "cos_apical_negg": [-0.05470547825098038, -0.3788420259952545, -0.39453125], "cos_Ac_negg": [-0.0008219259907491505, -0.27109235525131226, 0.4921875], "r_norm": [8.332502365112305, 8.355758666992188, 8.389439582824707]}, {"step": 3900, "epoch": 8, "train_loss": 8.812009811401367, "cos_r_negg": [-0.06844376027584076, -0.5706951022148132, -0.59765625], "cos_innovation_negg": [0.08148781210184097, -0.44096678495407104, 0.453125], "cos_apical_negg": [-0.06844376027584076, -0.5706951022148132, -0.59765625], "cos_Ac_negg": [0.0814904049038887, -0.4409674406051636, 0.453125], "r_norm": [8.454343795776367, 8.432502746582031, 8.447444915771484]}, {"step": 4000, "epoch": 8, "train_loss": 11.610506057739258, "cos_r_negg": [-0.0724836140871048, -0.7820213437080383, -0.8125], "cos_innovation_negg": [0.16154485940933228, -0.3440636694431305, 0.609375], "cos_apical_negg": [-0.0724836140871048, -0.7820213437080383, -0.8125], "cos_Ac_negg": [0.1615554839372635, -0.34406548738479614, 0.609375], "r_norm": [8.274002075195312, 8.270342826843262, 8.313014030456543], "ldr": {"dL_sdil": -4.135002136230469, "dL_bp": -4.135002136230469, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 8.431884765625, "cos_r_negg": [0.02188153564929962, -0.09907101094722748, -0.1015625], "cos_innovation_negg": [0.11786270886659622, 0.002072244882583618, 0.78515625], "cos_apical_negg": [0.02188153564929962, -0.09907101094722748, -0.1015625], "cos_Ac_negg": [0.11759955435991287, 0.0020715761929750443, 0.78515625], "r_norm": [8.24980354309082, 8.273475646972656, 8.269441604614258]}, {"step": 4200, "epoch": 8, "train_loss": 9.99185848236084, "cos_r_negg": [-0.06288471817970276, -0.5857329368591309, -0.61328125], "cos_innovation_negg": [-0.002537989290431142, -0.5164992809295654, 0.4375], "cos_apical_negg": [-0.06288471817970276, -0.5857329368591309, -0.61328125], "cos_Ac_negg": [-0.002540719462558627, -0.5164992213249207, 0.4375], "r_norm": [8.435721397399902, 8.411832809448242, 8.466842651367188]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.0958, "test_loss": 15.21412236328125}, {"step": 4300, "epoch": 9, "train_loss": 8.622159004211426, "cos_r_negg": [-0.19204388558864594, -0.7476558685302734, -0.78125], "cos_innovation_negg": [0.2395847886800766, -0.043641019612550735, 0.828125], "cos_apical_negg": [-0.19204388558864594, -0.7476558685302734, -0.78125], "cos_Ac_negg": [0.23957504332065582, -0.04364091902971268, 0.828125], "r_norm": [8.409965515136719, 8.437745094299316, 8.472014427185059]}, {"step": 4400, "epoch": 9, "train_loss": 10.624998092651367, "cos_r_negg": [0.012787628918886185, -0.5864592790603638, -0.61328125], "cos_innovation_negg": [-0.002128981752321124, -0.16572080552577972, 0.2734375], "cos_apical_negg": [0.012787628918886185, -0.5864592790603638, -0.61328125], "cos_Ac_negg": [-0.0022580455988645554, -0.16572090983390808, 0.2734375], "r_norm": [8.358915328979492, 8.35004997253418, 8.399492263793945]}, {"step": 4500, "epoch": 9, "train_loss": 9.67203140258789, "cos_r_negg": [0.0016504075611010194, 0.1448075771331787, 0.15234375], "cos_innovation_negg": [0.2078697830438614, -0.5442118644714355, 0.8359375], "cos_apical_negg": [0.0016504075611010194, 0.1448075771331787, 0.15234375], "cos_Ac_negg": [0.20786729454994202, -0.5442118048667908, 0.8359375], "r_norm": [7.790171146392822, 7.81021785736084, 7.82426643371582], "ldr": {"dL_sdil": -2.327272415161133, "dL_bp": -2.327272415161133, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 9.50709056854248, "cos_r_negg": [0.03133918344974518, -0.04891374707221985, -0.05078125], "cos_innovation_negg": [0.10768858343362808, -0.4323785901069641, 0.8359375], "cos_apical_negg": [0.03133918344974518, -0.04891374707221985, -0.05078125], "cos_Ac_negg": [0.10765126347541809, -0.43237853050231934, 0.8359375], "r_norm": [8.6129150390625, 8.625988006591797, 8.636297225952148]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.1032, "test_loss": 10.47324609375}, {"step": 4700, "epoch": 10, "train_loss": 17.704463958740234, "cos_r_negg": [-0.2113754153251648, -0.7924402952194214, -0.828125], "cos_innovation_negg": [-0.0420466884970665, -0.5955016613006592, 1.0], "cos_apical_negg": [-0.2113754153251648, -0.7924402952194214, -0.828125], "cos_Ac_negg": [-0.04216146096587181, -0.595502495765686, 1.0], "r_norm": [8.829102516174316, 8.858196258544922, 8.931103706359863]}, {"step": 4800, "epoch": 10, "train_loss": 13.77151107788086, "cos_r_negg": [-0.10815715789794922, -0.7763395309448242, -0.8125], "cos_innovation_negg": [-0.194504052400589, -0.6789554953575134, 0.828125], "cos_apical_negg": [-0.10815715789794922, -0.7763395309448242, -0.8125], "cos_Ac_negg": [-0.1944883018732071, -0.6789555549621582, 0.828125], "r_norm": [8.903640747070312, 8.928548812866211, 9.014246940612793]}, {"step": 4900, "epoch": 10, "train_loss": 12.760170936584473, "cos_r_negg": [0.093043252825737, -0.46575501561164856, -0.484375], "cos_innovation_negg": [-0.1991022378206253, -0.6289278268814087, 0.56640625], "cos_apical_negg": [0.093043252825737, -0.46575501561164856, -0.484375], "cos_Ac_negg": [-0.19910438358783722, -0.6289278864860535, 0.56640625], "r_norm": [8.481609344482422, 8.491607666015625, 8.50238037109375]}, {"step": 5000, "epoch": 10, "train_loss": 14.601521492004395, "cos_r_negg": [0.015325184911489487, -0.04902271926403046, -0.05078125], "cos_innovation_negg": [-0.008242804557085037, -0.9392389059066772, 0.6328125], "cos_apical_negg": [0.015325184911489487, -0.04902271926403046, -0.05078125], "cos_Ac_negg": [-0.008244137279689312, -0.9392389059066772, 0.6328125], "r_norm": [8.327488899230957, 8.346911430358887, 8.343986511230469], "ldr": {"dL_sdil": -1.96917724609375, "dL_bp": -1.96917724609375, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 18.300006866455078, "cos_r_negg": [-0.09675931930541992, -0.06755118817090988, -0.0703125], "cos_innovation_negg": [0.2105976790189743, 0.13013581931591034, 0.1484375], "cos_apical_negg": [-0.09675931930541992, -0.06755118817090988, -0.0703125], "cos_Ac_negg": [0.21060597896575928, 0.1301369071006775, 0.1484375], "r_norm": [7.851202487945557, 7.871732234954834, 7.909472465515137]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.1032, "test_loss": 8.86439248046875}, {"step": 5200, "epoch": 11, "train_loss": 17.184865951538086, "cos_r_negg": [-0.03285696730017662, -0.6279933452606201, -0.65625], "cos_innovation_negg": [-0.10381213575601578, -0.7768880724906921, 0.23046875], "cos_apical_negg": [-0.03285696730017662, -0.6279933452606201, -0.65625], "cos_Ac_negg": [-0.10381168127059937, -0.7768881320953369, 0.23046875], "r_norm": [9.077016830444336, 9.087455749511719, 9.146000862121582]}, {"step": 5300, "epoch": 11, "train_loss": 15.585555076599121, "cos_r_negg": [-0.054122310131788254, -0.25450626015663147, -0.265625], "cos_innovation_negg": [-0.21315738558769226, -0.4217783212661743, 0.46484375], "cos_apical_negg": [-0.054122310131788254, -0.25450626015663147, -0.265625], "cos_Ac_negg": [-0.21315713226795197, -0.42177826166152954, 0.46484375], "r_norm": [9.133200645446777, 9.143876075744629, 9.220969200134277]}, {"step": 5400, "epoch": 11, "train_loss": 11.582035064697266, "cos_r_negg": [-0.02124510146677494, -0.258146733045578, -0.26953125], "cos_innovation_negg": [-0.2423168271780014, -0.5489672422409058, 0.625], "cos_apical_negg": [-0.02124510146677494, -0.258146733045578, -0.26953125], "cos_Ac_negg": [-0.24231775104999542, -0.5489672422409058, 0.625], "r_norm": [8.897274017333984, 8.908099174499512, 8.955784797668457]}, {"step": 5500, "epoch": 11, "train_loss": 10.815536499023438, "cos_r_negg": [0.19488923251628876, 0.36498355865478516, 0.3828125], "cos_innovation_negg": [-0.11542116105556488, -0.5704564452171326, 0.796875], "cos_apical_negg": [0.19488923251628876, 0.36498355865478516, 0.3828125], "cos_Ac_negg": [-0.11541452258825302, -0.5704564452171326, 0.796875], "r_norm": [7.822964668273926, 7.8375749588012695, 7.815549850463867], "ldr": {"dL_sdil": -2.248777389526367, "dL_bp": -2.248777389526367, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 9.354345321655273, "cos_r_negg": [-0.19870108366012573, -0.4179230332374573, -0.4375], "cos_innovation_negg": [-0.19157151877880096, -0.5659834146499634, 0.29296875], "cos_apical_negg": [-0.19870108366012573, -0.4179230332374573, -0.4375], "cos_Ac_negg": [-0.1915706843137741, -0.5659831166267395, 0.29296875], "r_norm": [8.99805736541748, 9.00892448425293, 9.099859237670898]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.0892, "test_loss": 15.49762734375}, {"step": 5700, "epoch": 12, "train_loss": 12.49483871459961, "cos_r_negg": [0.032525885850191116, 0.10754494369029999, 0.11328125], "cos_innovation_negg": [-0.10293645411729813, -0.254874587059021, 0.52734375], "cos_apical_negg": [0.032525885850191116, 0.10754494369029999, 0.11328125], "cos_Ac_negg": [-0.10293140262365341, -0.25487449765205383, 0.52734375], "r_norm": [7.9246907234191895, 7.935434341430664, 7.929038047790527]}, {"step": 5800, "epoch": 12, "train_loss": 9.011554718017578, "cos_r_negg": [-0.1321522444486618, -0.40238243341445923, -0.421875], "cos_innovation_negg": [-0.12911565601825714, -0.4526625871658325, 1.0], "cos_apical_negg": [-0.1321522444486618, -0.40238243341445923, -0.421875], "cos_Ac_negg": [-0.13017389178276062, -0.4526626765727997, 1.0], "r_norm": [8.323644638061523, 8.33032512664795, 8.334144592285156]}, {"step": 5900, "epoch": 12, "train_loss": 7.745987415313721, "cos_r_negg": [-0.10914679616689682, -0.0486505962908268, -0.05078125], "cos_innovation_negg": [-0.19060179591178894, -0.37550675868988037, 0.6328125], "cos_apical_negg": [-0.10914679616689682, -0.0486505962908268, -0.05078125], "cos_Ac_negg": [-0.19060474634170532, -0.37550675868988037, 0.6328125], "r_norm": [8.682197570800781, 8.69565200805664, 8.712076187133789]}, {"step": 6000, "epoch": 12, "train_loss": 5.899158477783203, "cos_r_negg": [-0.056063584983348846, 0.3010885715484619, 0.31640625], "cos_innovation_negg": [-0.174213707447052, 0.010614042170345783, 0.73046875], "cos_apical_negg": [-0.056063584983348846, 0.3010885715484619, 0.31640625], "cos_Ac_negg": [-0.1742156744003296, 0.010614079423248768, 0.73046875], "r_norm": [8.285435676574707, 8.301414489746094, 8.360922813415527], "ldr": {"dL_sdil": -1.9572467803955078, "dL_bp": -1.9572467803955078, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.1135, "test_loss": 15.75487265625}, {"step": 6100, "epoch": 13, "train_loss": 13.229394912719727, "cos_r_negg": [-0.16768373548984528, -0.7887568473815918, -0.828125], "cos_innovation_negg": [-0.10204503685235977, -0.193100705742836, 1.0], "cos_apical_negg": [-0.16768373548984528, -0.7887568473815918, -0.828125], "cos_Ac_negg": [-0.10205547511577606, -0.1931007206439972, 1.0], "r_norm": [8.477532386779785, 8.494281768798828, 8.524208068847656]}, {"step": 6200, "epoch": 13, "train_loss": 9.279717445373535, "cos_r_negg": [-0.016502268612384796, -0.2530028223991394, -0.265625], "cos_innovation_negg": [-0.1391972303390503, -0.009444005787372589, 0.78515625], "cos_apical_negg": [-0.016502268612384796, -0.2530028223991394, -0.265625], "cos_Ac_negg": [-0.139200359582901, -0.009443961083889008, 0.78515625], "r_norm": [7.61728572845459, 7.632712364196777, 7.58719539642334]}, {"step": 6300, "epoch": 13, "train_loss": 15.58309555053711, "cos_r_negg": [-0.05130664259195328, -0.6103756427764893, -0.640625], "cos_innovation_negg": [-0.2856462895870209, 0.17206421494483948, 1.0], "cos_apical_negg": [-0.05130664259195328, -0.6103756427764893, -0.640625], "cos_Ac_negg": [-0.28565138578414917, 0.17206427454948425, 1.0], "r_norm": [7.944525241851807, 7.95833683013916, 7.917252540588379]}, {"step": 6400, "epoch": 13, "train_loss": 8.362821578979492, "cos_r_negg": [-0.12808667123317719, 0.10740907490253448, 0.11328125], "cos_innovation_negg": [-0.20652149617671967, -0.21156379580497742, 1.0], "cos_apical_negg": [-0.12808667123317719, 0.10740907490253448, 0.11328125], "cos_Ac_negg": [-0.20650528371334076, -0.21156372129917145, 1.0], "r_norm": [7.8779730796813965, 7.89633321762085, 7.933520793914795]}, {"step": 6500, "epoch": 13, "train_loss": 14.536314964294434, "cos_r_negg": [0.2417798936367035, 0.5563158988952637, 0.5859375], "cos_innovation_negg": [0.0997333899140358, 0.3189339339733124, 1.0], "cos_apical_negg": [0.2417798936367035, 0.5563158988952637, 0.5859375], "cos_Ac_negg": [0.09973182529211044, 0.31893399357795715, 1.0], "r_norm": [8.752015113830566, 8.779094696044922, 8.890295028686523], "ldr": {"dL_sdil": -2.398242950439453, "dL_bp": -2.398242950439453, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.0974, "test_loss": 14.60253017578125}, {"step": 6600, "epoch": 14, "train_loss": 10.260499000549316, "cos_r_negg": [0.1832551658153534, 0.5563303232192993, 0.5859375], "cos_innovation_negg": [-0.05300292372703552, 0.023668130859732628, 1.0], "cos_apical_negg": [0.1832551658153534, 0.5563303232192993, 0.5859375], "cos_Ac_negg": [-0.05303143709897995, 0.02366822212934494, 1.0], "r_norm": [8.428667068481445, 8.447489738464355, 8.54556941986084]}, {"step": 6700, "epoch": 14, "train_loss": 6.828721046447754, "cos_r_negg": [-0.05906118452548981, -0.4595968723297119, -0.484375], "cos_innovation_negg": [-0.27511993050575256, -0.14427368342876434, 0.40234375], "cos_apical_negg": [-0.05906118452548981, -0.4595968723297119, -0.484375], "cos_Ac_negg": [-0.27510368824005127, -0.14427348971366882, 0.40234375], "r_norm": [8.577927589416504, 8.599018096923828, 8.619671821594238]}, {"step": 6800, "epoch": 14, "train_loss": 7.994805812835693, "cos_r_negg": [-0.06416022032499313, -0.19613522291183472, -0.20703125], "cos_innovation_negg": [-0.10924974083900452, -0.42175769805908203, 0.84375], "cos_apical_negg": [-0.06416022032499313, -0.19613522291183472, -0.20703125], "cos_Ac_negg": [-0.10924537479877472, -0.42175793647766113, 0.84375], "r_norm": [8.493719100952148, 8.510171890258789, 8.519445419311523]}, {"step": 6900, "epoch": 14, "train_loss": 4.80057430267334, "cos_r_negg": [-0.08321009576320648, -0.3997562527656555, -0.421875], "cos_innovation_negg": [-0.04034553840756416, -0.05306457355618477, 1.0], "cos_apical_negg": [-0.08321009576320648, -0.3997562527656555, -0.421875], "cos_Ac_negg": [-0.04039899259805679, -0.05306461080908775, 1.0], "r_norm": [8.382085800170898, 8.395641326904297, 8.394950866699219]}, {"step": 7000, "epoch": 14, "train_loss": 12.29056167602539, "cos_r_negg": [-0.13885396718978882, 0.10711666941642761, 0.11328125], "cos_innovation_negg": [-0.17264874279499054, -0.26289743185043335, 0.73046875], "cos_apical_negg": [-0.13885396718978882, 0.10711666941642761, 0.11328125], "cos_Ac_negg": [-0.17285296320915222, -0.2628975212574005, 0.73046875], "r_norm": [8.34173583984375, 8.353206634521484, 8.408172607421875], "ldr": {"dL_sdil": -2.191899299621582, "dL_bp": -2.191899299621582, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1009, "test_loss": 13.5468400390625}], "final": {"test_acc": 0.1009, "test_loss": 13.5468400390625, "wall_s": 22.49405598640442, "cos_r_negg": [-0.15764665603637695, -0.40100422501564026, -0.421875], "cos_innovation_negg": [-0.2420637011528015, -0.1236443817615509, 0.84375], "cos_apical_negg": [-0.15764665603637695, -0.40100422501564026, -0.421875], "cos_Ac_negg": [-0.2420753389596939, -0.12364459037780762, 0.84375]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_raw_s3.json b/results/nuis_ctrl_v1_rho0p5_raw_s3.json new file mode 100644 index 0000000..86b899e --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_raw_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_raw_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.026506898924708366, 0.012330188415944576, -0.0019659455865621567], "cos_innovation_negg": [-0.029736431315541267, 0.019409991800785065, 0.008285761810839176], "cos_apical_negg": [-0.026506898924708366, 0.012330188415944576, -0.0019659455865621567], "cos_Ac_negg": [-0.02973800152540207, 0.019408803433179855, 0.008284433744847775], "r_norm": [8.17677116394043, 6.836240768432617, 6.476951599121094], "ldr": {"dL_sdil": 0.04908013343811035, "dL_bp": -0.21805977821350098, "ratio": -0.2250765081034628}}, {"step": 100, "epoch": 0, "train_loss": 12.267818450927734, "cos_r_negg": [-0.023786088451743126, -0.11871932446956635, -0.14084212481975555], "cos_innovation_negg": [-0.04167438670992851, -0.06051561236381531, 0.26192307472229004], "cos_apical_negg": [-0.023786088451743126, -0.11871932446956635, -0.14084212481975555], "cos_Ac_negg": [-0.041508134454488754, -0.06033655256032944, 0.26192396879196167], "r_norm": [10.945213317871094, 10.342334747314453, 9.944719314575195]}, {"step": 200, "epoch": 0, "train_loss": 11.568981170654297, "cos_r_negg": [0.05637676268815994, 0.16992853581905365, -0.49123942852020264], "cos_innovation_negg": [0.10289576649665833, 0.298967570066452, -0.0014939913526177406], "cos_apical_negg": [0.05637676268815994, 0.16992853581905365, -0.49123942852020264], "cos_Ac_negg": [0.10278133302927017, 0.29901963472366333, -0.0011754860170185566], "r_norm": [10.425918579101562, 9.996030807495117, 9.698972702026367]}, {"step": 300, "epoch": 0, "train_loss": 7.396692276000977, "cos_r_negg": [-0.027777038514614105, 0.0007287114858627319, -0.11273282766342163], "cos_innovation_negg": [-0.034390345215797424, 0.19405455887317657, 0.07537731528282166], "cos_apical_negg": [-0.027777038514614105, 0.0007287114858627319, -0.11273282766342163], "cos_Ac_negg": [-0.03439591825008392, 0.1940525770187378, 0.07536330819129944], "r_norm": [9.88321304321289, 9.525853157043457, 9.523195266723633]}, {"step": 400, "epoch": 0, "train_loss": 9.533720016479492, "cos_r_negg": [0.03617967665195465, 0.23808139562606812, -0.6518124938011169], "cos_innovation_negg": [0.03402910754084587, 0.2270115315914154, -0.4757543206214905], "cos_apical_negg": [0.03617967665195465, 0.23808139562606812, -0.6518124938011169], "cos_Ac_negg": [0.03401627391576767, 0.22700953483581543, -0.4757598042488098], "r_norm": [9.756996154785156, 9.257879257202148, 9.494220733642578]}, {"epoch_end": 0, "step": 469, "test_acc": 0.0982, "test_loss": 9.74996513671875}, {"step": 500, "epoch": 1, "train_loss": 16.520923614501953, "cos_r_negg": [-0.03965315967798233, 0.007813815027475357, -0.5233477354049683], "cos_innovation_negg": [-0.0692405104637146, -0.02429916523396969, -0.025975346565246582], "cos_apical_negg": [-0.03965315967798233, 0.007813815027475357, -0.5233477354049683], "cos_Ac_negg": [-0.06923192739486694, -0.02420998178422451, -0.025978829711675644], "r_norm": [9.435873031616211, 9.374765396118164, 9.238445281982422], "ldr": {"dL_sdil": -2.2517032623291016, "dL_bp": -2.2517032623291016, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 12.30771255493164, "cos_r_negg": [-0.015822438523173332, -0.1540403962135315, 0.08863641321659088], "cos_innovation_negg": [-0.01979876682162285, 0.10042668133974075, 0.3308795094490051], "cos_apical_negg": [-0.015822438523173332, -0.1540403962135315, 0.08863641321659088], "cos_Ac_negg": [-0.019798077642917633, 0.10043680667877197, 0.3308820128440857], "r_norm": [9.034561157226562, 8.978337287902832, 9.016508102416992]}, {"step": 700, "epoch": 1, "train_loss": 11.132923126220703, "cos_r_negg": [0.05001148581504822, -0.01987697370350361, 0.31429994106292725], "cos_innovation_negg": [0.05103549733757973, 0.1299055814743042, -0.21260008215904236], "cos_apical_negg": [0.05001148581504822, -0.01987697370350361, 0.31429994106292725], "cos_Ac_negg": [0.05102431774139404, 0.129903182387352, -0.21259984374046326], "r_norm": [9.006699562072754, 9.06875991821289, 9.216055870056152]}, {"step": 800, "epoch": 1, "train_loss": 8.47889518737793, "cos_r_negg": [0.005555157549679279, 0.047896090894937515, -0.29187458753585815], "cos_innovation_negg": [0.031460344791412354, 0.23891372978687286, -0.12157592177391052], "cos_apical_negg": [0.005555157549679279, 0.047896090894937515, -0.29187458753585815], "cos_Ac_negg": [0.03145395591855049, 0.23891803622245789, -0.12157764285802841], "r_norm": [8.915679931640625, 8.769262313842773, 9.054006576538086]}, {"step": 900, "epoch": 1, "train_loss": 14.730730056762695, "cos_r_negg": [-0.003456203965470195, -0.026968369260430336, -0.785609245300293], "cos_innovation_negg": [-0.047013331204652786, 0.16049107909202576, -0.6441376209259033], "cos_apical_negg": [-0.003456203965470195, -0.026968369260430336, -0.785609245300293], "cos_Ac_negg": [-0.047016218304634094, 0.16048306226730347, -0.6441439390182495], "r_norm": [9.154525756835938, 9.25345516204834, 9.408580780029297]}, {"epoch_end": 1, "step": 938, "test_acc": 0.1028, "test_loss": 12.37398525390625}, {"step": 1000, "epoch": 2, "train_loss": 14.343939781188965, "cos_r_negg": [0.0010967403650283813, -0.02762531489133835, -0.634750485420227], "cos_innovation_negg": [0.028234466910362244, 0.19616973400115967, 0.14262229204177856], "cos_apical_negg": [0.0010967403650283813, -0.02762531489133835, -0.634750485420227], "cos_Ac_negg": [0.028230516240000725, 0.1961737871170044, 0.14261820912361145], "r_norm": [8.693264961242676, 9.059747695922852, 8.956097602844238], "ldr": {"dL_sdil": -2.113520622253418, "dL_bp": -2.113520622253418, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 18.238525390625, "cos_r_negg": [0.0033974298276007175, 0.09077562391757965, -0.5978332161903381], "cos_innovation_negg": [-0.06636692583560944, 0.3485768735408783, -0.6810895204544067], "cos_apical_negg": [0.0033974298276007175, 0.09077562391757965, -0.5978332161903381], "cos_Ac_negg": [-0.06580479443073273, 0.35291773080825806, -0.6802278757095337], "r_norm": [8.441755294799805, 8.380655288696289, 8.638420104980469]}, {"step": 1200, "epoch": 2, "train_loss": 6.251697540283203, "cos_r_negg": [-0.030002865940332413, 0.09644234925508499, -0.017023786902427673], "cos_innovation_negg": [-0.00677952915430069, -0.27618831396102905, -0.1790367215871811], "cos_apical_negg": [-0.030002865940332413, 0.09644234925508499, -0.017023786902427673], "cos_Ac_negg": [-0.006759162992238998, -0.2757236361503601, -0.17903634905815125], "r_norm": [8.473408699035645, 8.536704063415527, 8.422385215759277]}, {"step": 1300, "epoch": 2, "train_loss": 12.23982048034668, "cos_r_negg": [0.007115112617611885, 0.019046638160943985, 0.3740012049674988], "cos_innovation_negg": [0.116559237241745, 0.37029099464416504, 0.29361745715141296], "cos_apical_negg": [0.007115112617611885, 0.019046638160943985, 0.3740012049674988], "cos_Ac_negg": [0.11655343323945999, 0.3702782690525055, 0.29361504316329956], "r_norm": [8.059736251831055, 8.151721000671387, 8.136381149291992]}, {"step": 1400, "epoch": 2, "train_loss": 13.769638061523438, "cos_r_negg": [0.003345026634633541, -0.1665327250957489, -0.30864810943603516], "cos_innovation_negg": [-0.006110879592597485, -0.050864942371845245, -0.40291962027549744], "cos_apical_negg": [0.003345026634633541, -0.1665327250957489, -0.30864810943603516], "cos_Ac_negg": [-0.0032902751117944717, -0.03094194084405899, -0.4010933041572571], "r_norm": [8.41011905670166, 8.431804656982422, 8.314620971679688]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.0974, "test_loss": 10.6078318359375}, {"step": 1500, "epoch": 3, "train_loss": 13.389451026916504, "cos_r_negg": [0.03407767415046692, -0.03753257542848587, -0.7653929591178894], "cos_innovation_negg": [0.026950674131512642, -0.4486401081085205, 0.10614737868309021], "cos_apical_negg": [0.03407767415046692, -0.03753257542848587, -0.7653929591178894], "cos_Ac_negg": [0.026941735297441483, -0.44864434003829956, 0.10614706575870514], "r_norm": [8.351848602294922, 8.403018951416016, 8.268065452575684], "ldr": {"dL_sdil": -2.9230241775512695, "dL_bp": -2.9230241775512695, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 13.374156951904297, "cos_r_negg": [-0.05378840118646622, -0.07085607945919037, 0.11260397732257843], "cos_innovation_negg": [-0.021177195012569427, -0.14570271968841553, 0.042436107993125916], "cos_apical_negg": [-0.05378840118646622, -0.07085607945919037, 0.11260397732257843], "cos_Ac_negg": [-0.021177995949983597, -0.14570161700248718, 0.04243840277194977], "r_norm": [8.2179594039917, 8.293608665466309, 8.363481521606445]}, {"step": 1700, "epoch": 3, "train_loss": 7.247921943664551, "cos_r_negg": [0.0029886290431022644, -0.4298208951950073, -0.6133275628089905], "cos_innovation_negg": [-0.0890326201915741, -0.04581644386053085, -0.14729204773902893], "cos_apical_negg": [0.0029886290431022644, -0.4298208951950073, -0.6133275628089905], "cos_Ac_negg": [-0.0890505313873291, -0.045822739601135254, -0.14729176461696625], "r_norm": [8.244298934936523, 8.475659370422363, 8.392572402954102]}, {"step": 1800, "epoch": 3, "train_loss": 11.349238395690918, "cos_r_negg": [0.026772530749440193, -0.4606841206550598, -0.45862460136413574], "cos_innovation_negg": [0.052183251827955246, -0.2358827143907547, -0.1804431974887848], "cos_apical_negg": [0.026772530749440193, -0.4606841206550598, -0.45862460136413574], "cos_Ac_negg": [0.05213166028261185, -0.23588411509990692, -0.18044695258140564], "r_norm": [8.370306015014648, 8.345922470092773, 8.283858299255371]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.1032, "test_loss": 10.46290751953125}, {"step": 1900, "epoch": 4, "train_loss": 11.21202278137207, "cos_r_negg": [0.0524648018181324, -0.48226457834243774, -0.457119882106781], "cos_innovation_negg": [0.07819177210330963, -0.024834981188178062, 0.22201880812644958], "cos_apical_negg": [0.0524648018181324, -0.48226457834243774, -0.457119882106781], "cos_Ac_negg": [0.07818964123725891, -0.02483757957816124, 0.2220175713300705], "r_norm": [8.23788070678711, 8.438673973083496, 8.403940200805664]}, {"step": 2000, "epoch": 4, "train_loss": 12.917704582214355, "cos_r_negg": [0.024774182587862015, -0.35545337200164795, -0.2560240924358368], "cos_innovation_negg": [0.1487734168767929, -0.35608553886413574, 0.31173187494277954], "cos_apical_negg": [0.024774182587862015, -0.35545337200164795, -0.2560240924358368], "cos_Ac_negg": [0.14858537912368774, -0.3560815751552582, 0.3117333948612213], "r_norm": [8.501142501831055, 8.462979316711426, 8.409832000732422], "ldr": {"dL_sdil": -2.218583106994629, "dL_bp": -2.2185840606689453, "ratio": 0.9999995701428072}}, {"step": 2100, "epoch": 4, "train_loss": 7.7389984130859375, "cos_r_negg": [-0.00706346845254302, -0.2522203028202057, -0.6268371343612671], "cos_innovation_negg": [0.14600668847560883, -0.01675611361861229, 0.6800406575202942], "cos_apical_negg": [-0.00706346845254302, -0.2522203028202057, -0.6268371343612671], "cos_Ac_negg": [0.14601609110832214, -0.016754742711782455, 0.6800392866134644], "r_norm": [7.930294036865234, 7.923968315124512, 7.821818828582764]}, {"step": 2200, "epoch": 4, "train_loss": 20.193058013916016, "cos_r_negg": [-0.00441304175183177, 0.27741676568984985, 0.1075911819934845], "cos_innovation_negg": [0.24156762659549713, -0.19210182130336761, 0.38424548506736755], "cos_apical_negg": [-0.00441304175183177, 0.27741676568984985, 0.1075911819934845], "cos_Ac_negg": [0.24169965088367462, -0.19211789965629578, 0.3842449188232422], "r_norm": [8.333871841430664, 8.394327163696289, 8.443845748901367]}, {"step": 2300, "epoch": 4, "train_loss": 13.372560501098633, "cos_r_negg": [-0.01916484721004963, -0.3653978705406189, -0.22849704325199127], "cos_innovation_negg": [-0.2404419183731079, -0.1987619400024414, 0.31692421436309814], "cos_apical_negg": [-0.01916484721004963, -0.3653978705406189, -0.22849704325199127], "cos_Ac_negg": [-0.24046073853969574, -0.1987646222114563, 0.31692367792129517], "r_norm": [8.675909996032715, 8.760309219360352, 8.730667114257812]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.101, "test_loss": 18.4356232421875}, {"step": 2400, "epoch": 5, "train_loss": 13.460979461669922, "cos_r_negg": [0.05073873698711395, -0.4613088369369507, -0.7670339345932007], "cos_innovation_negg": [-0.17681385576725006, -0.19277063012123108, -0.050577402114868164], "cos_apical_negg": [0.05073873698711395, -0.4613088369369507, -0.7670339345932007], "cos_Ac_negg": [-0.1768469512462616, -0.1927729845046997, -0.05057763308286667], "r_norm": [8.736212730407715, 8.846563339233398, 8.852428436279297]}, {"step": 2500, "epoch": 5, "train_loss": 14.70837688446045, "cos_r_negg": [-0.09024171531200409, -0.4535521864891052, -0.7785865664482117], "cos_innovation_negg": [-0.12258277088403702, -0.23267489671707153, -0.24728801846504211], "cos_apical_negg": [-0.09024171531200409, -0.4535521864891052, -0.7785865664482117], "cos_Ac_negg": [-0.1221993938088417, -0.23265603184700012, -0.24729013442993164], "r_norm": [8.82823371887207, 8.948509216308594, 9.067193984985352], "ldr": {"dL_sdil": -3.5159425735473633, "dL_bp": -3.5159425735473633, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 11.377114295959473, "cos_r_negg": [0.04751364886760712, -0.4140342175960541, -0.279313862323761], "cos_innovation_negg": [-0.14037027955055237, 0.0013605942949652672, -0.02792779728770256], "cos_apical_negg": [0.04751364886760712, -0.4140342175960541, -0.279313862323761], "cos_Ac_negg": [-0.14030057191848755, 0.0013641472905874252, -0.027927536517381668], "r_norm": [8.877184867858887, 8.897836685180664, 9.025771141052246]}, {"step": 2700, "epoch": 5, "train_loss": 8.009690284729004, "cos_r_negg": [-0.0716434195637703, -0.49250704050064087, -0.27448514103889465], "cos_innovation_negg": [-0.1671718955039978, 0.33916231989860535, 0.3194931745529175], "cos_apical_negg": [-0.0716434195637703, -0.49250704050064087, -0.27448514103889465], "cos_Ac_negg": [-0.16718293726444244, 0.3391610383987427, 0.3194931745529175], "r_norm": [8.489503860473633, 8.43554973602295, 8.49609661102295]}, {"step": 2800, "epoch": 5, "train_loss": 11.573670387268066, "cos_r_negg": [0.036237310618162155, -0.4692964255809784, 0.0005364846438169479], "cos_innovation_negg": [-0.02590039372444153, 0.4003668427467346, 0.13431602716445923], "cos_apical_negg": [0.036237310618162155, -0.4692964255809784, 0.0005364846438169479], "cos_Ac_negg": [-0.025902483612298965, 0.40036606788635254, 0.1343161165714264], "r_norm": [8.71239185333252, 8.669532775878906, 8.771892547607422]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.1028, "test_loss": 6.336495654296875}, {"step": 2900, "epoch": 6, "train_loss": 8.879903793334961, "cos_r_negg": [0.015899918973445892, -0.15473672747612, 0.2709038257598877], "cos_innovation_negg": [0.13754628598690033, -0.036785535514354706, 0.735919177532196], "cos_apical_negg": [0.015899918973445892, -0.15473672747612, 0.2709038257598877], "cos_Ac_negg": [0.1375868171453476, -0.036786727607250214, 0.7359188199043274], "r_norm": [8.138741493225098, 8.189132690429688, 8.175435066223145]}, {"step": 3000, "epoch": 6, "train_loss": 9.869020462036133, "cos_r_negg": [-0.14766976237297058, 0.29564493894577026, -0.11502531170845032], "cos_innovation_negg": [-0.1497650444507599, -0.05764322727918625, 0.6482102870941162], "cos_apical_negg": [-0.14766976237297058, 0.29564493894577026, -0.11502531170845032], "cos_Ac_negg": [-0.14863978326320648, -0.05764535069465637, 0.6482106447219849], "r_norm": [8.702150344848633, 8.741813659667969, 8.820833206176758], "ldr": {"dL_sdil": -2.196687698364258, "dL_bp": -2.196687698364258, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 14.6121244430542, "cos_r_negg": [0.10387705266475677, -0.4871651232242584, -0.34501147270202637], "cos_innovation_negg": [0.152975931763649, -0.02190149389207363, 0.4271950125694275], "cos_apical_negg": [0.10387705266475677, -0.4871651232242584, -0.34501147270202637], "cos_Ac_negg": [0.15296901762485504, -0.021896515041589737, 0.42719489336013794], "r_norm": [7.911396026611328, 7.8898491859436035, 7.8567962646484375]}, {"step": 3200, "epoch": 6, "train_loss": 17.088899612426758, "cos_r_negg": [0.04362340644001961, -0.35752764344215393, -0.15494117140769958], "cos_innovation_negg": [0.015425674617290497, -0.04150226712226868, 0.4982662796974182], "cos_apical_negg": [0.04362340644001961, -0.35752764344215393, -0.15494117140769958], "cos_Ac_negg": [0.015409242361783981, -0.04149831086397171, 0.4982661008834839], "r_norm": [7.96759557723999, 7.954899787902832, 7.929862022399902]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.098, "test_loss": 9.185275}, {"step": 3300, "epoch": 7, "train_loss": 7.184060573577881, "cos_r_negg": [-0.1503773182630539, -0.3704270124435425, -0.7313346862792969], "cos_innovation_negg": [0.06171245127916336, -0.06041629612445831, 0.04990876838564873], "cos_apical_negg": [-0.1503773182630539, -0.3704270124435425, -0.7313346862792969], "cos_Ac_negg": [0.061731692403554916, -0.06041664630174637, 0.04990893229842186], "r_norm": [8.458047866821289, 8.463258743286133, 8.480491638183594]}, {"step": 3400, "epoch": 7, "train_loss": 11.476707458496094, "cos_r_negg": [-0.18878662586212158, -0.2959175109863281, -0.1998378485441208], "cos_innovation_negg": [0.006415966432541609, 0.10584098100662231, 0.6804267764091492], "cos_apical_negg": [-0.18878662586212158, -0.2959175109863281, -0.1998378485441208], "cos_Ac_negg": [0.006680776830762625, 0.10584947466850281, 0.6804301142692566], "r_norm": [8.060136795043945, 8.106498718261719, 8.083683967590332]}, {"step": 3500, "epoch": 7, "train_loss": 14.253765106201172, "cos_r_negg": [-0.14449641108512878, -0.4677894711494446, -0.5634869337081909], "cos_innovation_negg": [-0.19055455923080444, -0.10068078339099884, 0.013241913169622421], "cos_apical_negg": [-0.14449641108512878, -0.4677894711494446, -0.5634869337081909], "cos_Ac_negg": [-0.19058990478515625, -0.10068131238222122, 0.013241893611848354], "r_norm": [8.630552291870117, 8.639185905456543, 8.669866561889648], "ldr": {"dL_sdil": -2.0724716186523438, "dL_bp": -2.0724716186523438, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 7.054471969604492, "cos_r_negg": [-0.16243915259838104, -0.024647122249007225, -0.12824252247810364], "cos_innovation_negg": [0.06638328731060028, 0.022071314975619316, 0.4062129557132721], "cos_apical_negg": [-0.16243915259838104, -0.024647122249007225, -0.12824252247810364], "cos_Ac_negg": [0.06640221923589706, 0.022071365267038345, 0.40621307492256165], "r_norm": [7.919602394104004, 7.961474418640137, 7.932734489440918]}, {"step": 3700, "epoch": 7, "train_loss": 11.617191314697266, "cos_r_negg": [-0.06732095777988434, -0.3359779715538025, -0.6818439364433289], "cos_innovation_negg": [0.23047637939453125, 0.2817847728729248, 0.6891072392463684], "cos_apical_negg": [-0.06732095777988434, -0.3359779715538025, -0.6818439364433289], "cos_Ac_negg": [0.23048265278339386, 0.2817847728729248, 0.6891072988510132], "r_norm": [7.801712512969971, 7.833863258361816, 7.814785957336426]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.1135, "test_loss": 12.69340283203125}, {"step": 3800, "epoch": 8, "train_loss": 10.240337371826172, "cos_r_negg": [-0.17147889733314514, -0.5333845019340515, -0.3955014944076538], "cos_innovation_negg": [-0.2837035655975342, -0.08730654418468475, 0.22334203124046326], "cos_apical_negg": [-0.17147889733314514, -0.5333845019340515, -0.3955014944076538], "cos_Ac_negg": [-0.28373071551322937, -0.08730682730674744, 0.22334203124046326], "r_norm": [8.771690368652344, 8.787416458129883, 8.83869743347168]}, {"step": 3900, "epoch": 8, "train_loss": 10.28286361694336, "cos_r_negg": [-0.08871660381555557, -0.5282332897186279, -0.2800084948539734], "cos_innovation_negg": [0.01278554555028677, 0.2824883460998535, 0.6813057065010071], "cos_apical_negg": [-0.08871660381555557, -0.5282332897186279, -0.2800084948539734], "cos_Ac_negg": [0.012820485047996044, 0.2824826240539551, 0.6813047528266907], "r_norm": [7.617300033569336, 7.614985942840576, 7.571664333343506]}, {"step": 4000, "epoch": 8, "train_loss": 20.491256713867188, "cos_r_negg": [-0.28162068128585815, -0.35697758197784424, -0.21034105122089386], "cos_innovation_negg": [-0.03233714774250984, 0.25930458307266235, 0.7615891695022583], "cos_apical_negg": [-0.28162068128585815, -0.35697758197784424, -0.21034105122089386], "cos_Ac_negg": [-0.032187316566705704, 0.25930604338645935, 0.7615891695022583], "r_norm": [7.838986396789551, 7.877105236053467, 7.871551990509033], "ldr": {"dL_sdil": -2.564411163330078, "dL_bp": -2.564411163330078, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 5.977367401123047, "cos_r_negg": [-0.17211070656776428, -0.4709746241569519, -0.6017636060714722], "cos_innovation_negg": [-0.22096621990203857, -0.29795733094215393, 0.06907035410404205], "cos_apical_negg": [-0.17211070656776428, -0.4709746241569519, -0.6017636060714722], "cos_Ac_negg": [-0.22096644341945648, -0.2979571521282196, 0.06906933337450027], "r_norm": [8.719335556030273, 8.767009735107422, 8.825902938842773]}, {"step": 4200, "epoch": 8, "train_loss": 12.024679183959961, "cos_r_negg": [-0.21996182203292847, -0.3851681351661682, -0.2526507079601288], "cos_innovation_negg": [0.17134536802768707, 0.2002130001783371, 0.8018349409103394], "cos_apical_negg": [-0.21996182203292847, -0.3851681351661682, -0.2526507079601288], "cos_Ac_negg": [0.17138904333114624, 0.20021295547485352, 0.8018350005149841], "r_norm": [8.059459686279297, 8.085622787475586, 8.084490776062012]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.098, "test_loss": 11.16186103515625}, {"step": 4300, "epoch": 9, "train_loss": 14.091529846191406, "cos_r_negg": [-0.19059602916240692, -0.3642621338367462, -0.7494282722473145], "cos_innovation_negg": [-0.11106379330158234, -0.33209705352783203, 0.9318732023239136], "cos_apical_negg": [-0.19059602916240692, -0.3642621338367462, -0.7494282722473145], "cos_Ac_negg": [-0.1656332015991211, -0.3321491479873657, 0.9318636059761047], "r_norm": [8.805402755737305, 8.811711311340332, 8.850028991699219]}, {"step": 4400, "epoch": 9, "train_loss": 7.883824348449707, "cos_r_negg": [-0.0642743781208992, -0.488760769367218, -0.5124220252037048], "cos_innovation_negg": [-0.0667971819639206, 0.3573368787765503, 0.37410908937454224], "cos_apical_negg": [-0.0642743781208992, -0.488760769367218, -0.5124220252037048], "cos_Ac_negg": [-0.06663763523101807, 0.35733675956726074, 0.3741092085838318], "r_norm": [7.717794418334961, 7.725761413574219, 7.704934597015381]}, {"step": 4500, "epoch": 9, "train_loss": 6.33135986328125, "cos_r_negg": [-0.23611389100551605, -0.47508981823921204, -0.5680159330368042], "cos_innovation_negg": [-0.2481411248445511, -0.4730244278907776, 0.14018017053604126], "cos_apical_negg": [-0.23611389100551605, -0.47508981823921204, -0.5680159330368042], "cos_Ac_negg": [-0.24856628477573395, -0.47302454710006714, 0.14015567302703857], "r_norm": [8.927231788635254, 8.9688138961792, 9.046774864196777], "ldr": {"dL_sdil": -4.615302085876465, "dL_bp": -4.615302085876465, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 10.78182601928711, "cos_r_negg": [0.07771191745996475, -0.22075295448303223, 0.5076983571052551], "cos_innovation_negg": [-0.09848885983228683, -0.1406552791595459, 0.7602018117904663], "cos_apical_negg": [0.07771191745996475, -0.22075295448303223, 0.5076983571052551], "cos_Ac_negg": [-0.10116881877183914, -0.1406559944152832, 0.7602018117904663], "r_norm": [8.72419548034668, 8.749523162841797, 8.857084274291992]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.1135, "test_loss": 12.2586244140625}, {"step": 4700, "epoch": 10, "train_loss": 11.82200813293457, "cos_r_negg": [-0.0790974572300911, -0.5411538481712341, -0.17489510774612427], "cos_innovation_negg": [0.22131283581256866, 0.46562597155570984, 0.503681480884552], "cos_apical_negg": [-0.0790974572300911, -0.5411538481712341, -0.17489510774612427], "cos_Ac_negg": [0.2213067263364792, 0.4656198024749756, 0.5036664605140686], "r_norm": [8.224967956542969, 8.225711822509766, 8.285591125488281]}, {"step": 4800, "epoch": 10, "train_loss": 6.89193058013916, "cos_r_negg": [-0.0241834819316864, -0.1250014454126358, 0.565407395362854], "cos_innovation_negg": [0.04167366027832031, -0.07039815187454224, 0.8029267191886902], "cos_apical_negg": [-0.0241834819316864, -0.1250014454126358, 0.565407395362854], "cos_Ac_negg": [0.0416577123105526, -0.07039816677570343, 0.8029266595840454], "r_norm": [9.021595001220703, 9.044514656066895, 9.143796920776367]}, {"step": 4900, "epoch": 10, "train_loss": 14.364617347717285, "cos_r_negg": [-0.09531091153621674, -0.360135555267334, -0.3966771364212036], "cos_innovation_negg": [-0.07605627179145813, 0.12930813431739807, 0.842853307723999], "cos_apical_negg": [-0.09531091153621674, -0.360135555267334, -0.3966771364212036], "cos_Ac_negg": [-0.07411383092403412, 0.12930810451507568, 0.842853307723999], "r_norm": [7.924593925476074, 7.934759140014648, 7.925215721130371]}, {"step": 5000, "epoch": 10, "train_loss": 14.163548469543457, "cos_r_negg": [0.14270110428333282, 0.26727262139320374, -0.24067597091197968], "cos_innovation_negg": [0.2650975286960602, 0.09857301414012909, 0.46744969487190247], "cos_apical_negg": [0.14270110428333282, 0.26727262139320374, -0.24067597091197968], "cos_Ac_negg": [0.2659301459789276, 0.09856803715229034, 0.46744999289512634], "r_norm": [8.169264793395996, 8.181272506713867, 8.210319519042969], "ldr": {"dL_sdil": -3.5037050247192383, "dL_bp": -3.5037050247192383, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 13.023917198181152, "cos_r_negg": [-0.35690081119537354, -0.47259896993637085, -0.6792689561843872], "cos_innovation_negg": [-0.41525083780288696, -0.3760499954223633, 0.219245046377182], "cos_apical_negg": [-0.35690081119537354, -0.47259896993637085, -0.6792689561843872], "cos_Ac_negg": [-0.41535335779190063, -0.37605011463165283, 0.219245046377182], "r_norm": [9.015456199645996, 9.048063278198242, 9.120501518249512]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.0958, "test_loss": 8.358147314453126}, {"step": 5200, "epoch": 11, "train_loss": 9.056848526000977, "cos_r_negg": [0.143497496843338, 0.29202643036842346, 0.40848076343536377], "cos_innovation_negg": [0.20552386343479156, 0.1962636113166809, 0.8469909429550171], "cos_apical_negg": [0.143497496843338, 0.29202643036842346, 0.40848076343536377], "cos_Ac_negg": [0.20551790297031403, 0.1962636113166809, 0.8469908237457275], "r_norm": [8.368833541870117, 8.382272720336914, 8.390267372131348]}, {"step": 5300, "epoch": 11, "train_loss": 10.967495918273926, "cos_r_negg": [-0.33753615617752075, -0.3109343647956848, -0.7291955947875977], "cos_innovation_negg": [-0.012439723126590252, -0.03944985568523407, 0.9048149585723877], "cos_apical_negg": [-0.33753615617752075, -0.3109343647956848, -0.7291955947875977], "cos_Ac_negg": [-0.012433229945600033, -0.039450086653232574, 0.9048150777816772], "r_norm": [8.316133499145508, 8.330072402954102, 8.333383560180664]}, {"step": 5400, "epoch": 11, "train_loss": 13.981063842773438, "cos_r_negg": [-0.15270976722240448, -0.41802653670310974, -0.41921016573905945], "cos_innovation_negg": [-0.000549065473023802, -0.3085049092769623, 0.4921051561832428], "cos_apical_negg": [-0.15270976722240448, -0.41802653670310974, -0.41921016573905945], "cos_Ac_negg": [-0.0007135692867450416, -0.30850499868392944, 0.4921052157878876], "r_norm": [8.57032585144043, 8.574323654174805, 8.580300331115723]}, {"step": 5500, "epoch": 11, "train_loss": 13.959887504577637, "cos_r_negg": [-0.27541258931159973, -0.34256115555763245, -0.7369647026062012], "cos_innovation_negg": [0.23313891887664795, 0.075493223965168, 0.9400489330291748], "cos_apical_negg": [-0.27541258931159973, -0.34256115555763245, -0.7369647026062012], "cos_Ac_negg": [0.23306874930858612, 0.07549314200878143, 0.9400489330291748], "r_norm": [8.232297897338867, 8.24512767791748, 8.244892120361328], "ldr": {"dL_sdil": -2.124760627746582, "dL_bp": -2.124760627746582, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 9.787443161010742, "cos_r_negg": [-0.3273448944091797, -0.49620890617370605, -0.6517419219017029], "cos_innovation_negg": [-0.3757983446121216, -0.3121603727340698, 0.40261226892471313], "cos_apical_negg": [-0.3273448944091797, -0.49620890617370605, -0.6517419219017029], "cos_Ac_negg": [-0.37583214044570923, -0.3121604323387146, 0.40261223912239075], "r_norm": [8.811349868774414, 8.843202590942383, 8.900054931640625]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.098, "test_loss": 11.8320888671875}, {"step": 5700, "epoch": 12, "train_loss": 15.085618019104004, "cos_r_negg": [-0.25524723529815674, -0.3805316388607025, -0.2532828450202942], "cos_innovation_negg": [0.2877015769481659, 0.34255295991897583, 0.5990843176841736], "cos_apical_negg": [-0.25524723529815674, -0.3805316388607025, -0.2532828450202942], "cos_Ac_negg": [0.28769752383232117, 0.3425529897212982, 0.5990841388702393], "r_norm": [7.480499744415283, 7.491408348083496, 7.479680061340332]}, {"step": 5800, "epoch": 12, "train_loss": 7.651779651641846, "cos_r_negg": [-0.2340777963399887, -0.36308926343917847, -0.2652818560600281], "cos_innovation_negg": [0.22504860162734985, -0.03697636350989342, 0.6941032409667969], "cos_apical_negg": [-0.2340777963399887, -0.36308926343917847, -0.2652818560600281], "cos_Ac_negg": [0.22500714659690857, -0.03697629272937775, 0.6941028833389282], "r_norm": [8.390066146850586, 8.395925521850586, 8.41372299194336]}, {"step": 5900, "epoch": 12, "train_loss": 9.20200252532959, "cos_r_negg": [-0.19804447889328003, -0.2335311472415924, -0.022346118465065956], "cos_innovation_negg": [0.2170308530330658, 0.017645228654146194, 0.7609648704528809], "cos_apical_negg": [-0.19804447889328003, -0.2335311472415924, -0.022346118465065956], "cos_Ac_negg": [0.21703936159610748, 0.017645198851823807, 0.7609648704528809], "r_norm": [8.290702819824219, 8.303144454956055, 8.30451774597168]}, {"step": 6000, "epoch": 12, "train_loss": 8.42834186553955, "cos_r_negg": [-0.1310756653547287, -0.42125949263572693, -0.49664759635925293], "cos_innovation_negg": [0.05727000907063484, -0.1939908117055893, 0.49494606256484985], "cos_apical_negg": [-0.1310756653547287, -0.42125949263572693, -0.49664759635925293], "cos_Ac_negg": [0.05721993371844292, -0.1939908117055893, 0.4949462413787842], "r_norm": [8.714231491088867, 8.73741626739502, 8.776906967163086], "ldr": {"dL_sdil": -2.2645034790039062, "dL_bp": -2.2645034790039062, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.0982, "test_loss": 8.90008359375}, {"step": 6100, "epoch": 13, "train_loss": 10.190654754638672, "cos_r_negg": [-0.131789892911911, -0.34542152285575867, -0.577663004398346], "cos_innovation_negg": [0.32587963342666626, 0.2613799571990967, 0.9931835532188416], "cos_apical_negg": [-0.131789892911911, -0.34542152285575867, -0.577663004398346], "cos_Ac_negg": [0.3258972465991974, 0.26138001680374146, 0.9931835532188416], "r_norm": [8.087332725524902, 8.10606861114502, 8.103015899658203]}, {"step": 6200, "epoch": 13, "train_loss": 15.763067245483398, "cos_r_negg": [-0.22160448133945465, -0.44533249735832214, -0.5738877058029175], "cos_innovation_negg": [-0.34329837560653687, -0.5508235692977905, 0.3571058511734009], "cos_apical_negg": [-0.22160448133945465, -0.44533249735832214, -0.5738877058029175], "cos_Ac_negg": [-0.3432968854904175, -0.5508233308792114, 0.35710570216178894], "r_norm": [8.938228607177734, 8.966825485229492, 9.043449401855469]}, {"step": 6300, "epoch": 13, "train_loss": 7.8466410636901855, "cos_r_negg": [-0.29713305830955505, -0.3718841075897217, -0.7026219964027405], "cos_innovation_negg": [0.2911476492881775, 0.14079971611499786, 0.8674792051315308], "cos_apical_negg": [-0.29713305830955505, -0.3718841075897217, -0.7026219964027405], "cos_Ac_negg": [0.2911386489868164, 0.1407995969057083, 0.8674792051315308], "r_norm": [8.247562408447266, 8.26615047454834, 8.277181625366211]}, {"step": 6400, "epoch": 13, "train_loss": 14.184345245361328, "cos_r_negg": [-0.34605881571769714, -0.3860695958137512, -0.728797197341919], "cos_innovation_negg": [0.3147152364253998, 0.32621294260025024, 0.8966970443725586], "cos_apical_negg": [-0.34605881571769714, -0.3860695958137512, -0.728797197341919], "cos_Ac_negg": [0.31331801414489746, 0.3262106776237488, 0.8966970443725586], "r_norm": [8.174250602722168, 8.199581146240234, 8.235008239746094]}, {"step": 6500, "epoch": 13, "train_loss": 11.187944412231445, "cos_r_negg": [-0.0576140470802784, -0.3494236469268799, -0.27806684374809265], "cos_innovation_negg": [-0.009291226044297218, 0.36055776476860046, 0.7719247341156006], "cos_apical_negg": [-0.0576140470802784, -0.3494236469268799, -0.27806684374809265], "cos_Ac_negg": [-0.009204496629536152, 0.36056098341941833, 0.7719250917434692], "r_norm": [8.064088821411133, 8.076695442199707, 8.132064819335938], "ldr": {"dL_sdil": -6.145732879638672, "dL_bp": -6.145732879638672, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.0982, "test_loss": 12.07663271484375}, {"step": 6600, "epoch": 14, "train_loss": 7.594141006469727, "cos_r_negg": [-0.143095925450325, -0.2919423580169678, -0.12109537422657013], "cos_innovation_negg": [0.13657832145690918, 0.01721436157822609, 0.7777774930000305], "cos_apical_negg": [-0.143095925450325, -0.2919423580169678, -0.12109537422657013], "cos_Ac_negg": [0.13656969368457794, 0.01721443422138691, 0.7777774930000305], "r_norm": [8.260603904724121, 8.273170471191406, 8.27216911315918]}, {"step": 6700, "epoch": 14, "train_loss": 10.048295974731445, "cos_r_negg": [-0.31014254689216614, -0.4719916582107544, -0.7264832258224487], "cos_innovation_negg": [0.013995213434100151, 0.061046868562698364, 0.6761834621429443], "cos_apical_negg": [-0.31014254689216614, -0.4719916582107544, -0.7264832258224487], "cos_Ac_negg": [0.014052307233214378, 0.06104706972837448, 0.6761834621429443], "r_norm": [8.349325180053711, 8.3685302734375, 8.412313461303711]}, {"step": 6800, "epoch": 14, "train_loss": 8.724132537841797, "cos_r_negg": [-0.03416309133172035, -0.15668511390686035, 0.019666921347379684], "cos_innovation_negg": [-0.02465755119919777, 0.025783326476812363, 0.9365755915641785], "cos_apical_negg": [-0.03416309133172035, -0.15668511390686035, 0.019666921347379684], "cos_Ac_negg": [-0.02466672472655773, 0.0257832370698452, 0.9365756511688232], "r_norm": [8.282548904418945, 8.297465324401855, 8.315971374511719]}, {"step": 6900, "epoch": 14, "train_loss": 14.09990119934082, "cos_r_negg": [-0.255214661359787, -0.3579021096229553, -0.4531940817832947], "cos_innovation_negg": [0.328462690114975, 0.20817600190639496, 0.8455662727355957], "cos_apical_negg": [-0.255214661359787, -0.3579021096229553, -0.4531940817832947], "cos_Ac_negg": [0.3284487724304199, 0.20817597210407257, 0.8455663919448853], "r_norm": [8.243138313293457, 8.261075973510742, 8.279829025268555]}, {"step": 7000, "epoch": 14, "train_loss": 8.938194274902344, "cos_r_negg": [0.08056957274675369, -0.14365482330322266, 0.5462514758110046], "cos_innovation_negg": [0.21136142313480377, -0.13913632929325104, 0.8379552364349365], "cos_apical_negg": [0.08056957274675369, -0.14365482330322266, 0.5462514758110046], "cos_Ac_negg": [0.21127372980117798, -0.13913647830486298, 0.8379553556442261], "r_norm": [8.701251983642578, 8.723127365112305, 8.845736503601074], "ldr": {"dL_sdil": -2.529360771179199, "dL_bp": -2.529360771179199, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1028, "test_loss": 8.322147998046875}], "final": {"test_acc": 0.1028, "test_loss": 8.322147998046875, "wall_s": 22.142099618911743, "cos_r_negg": [-0.2482898086309433, -0.316217303276062, -0.7253843545913696], "cos_innovation_negg": [0.15539732575416565, 0.10566452890634537, 0.8286952972412109], "cos_apical_negg": [-0.2482898086309433, -0.316217303276062, -0.7253843545913696], "cos_Ac_negg": [0.15538889169692993, 0.10566452145576477, 0.8286952972412109]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_raw_s4.json b/results/nuis_ctrl_v1_rho0p5_raw_s4.json new file mode 100644 index 0000000..38e1b3a --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_raw_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 0, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_raw_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.009717854671180248, 0.0016548233106732368, -0.010080229490995407], "cos_innovation_negg": [-0.00785096175968647, 0.011800158768892288, 0.006814056076109409], "cos_apical_negg": [-0.009717854671180248, 0.0016548233106732368, -0.010080229490995407], "cos_Ac_negg": [-0.007853087037801743, 0.011798959225416183, 0.006812635809183121], "r_norm": [8.169258117675781, 6.983946323394775, 6.580234527587891], "ldr": {"dL_sdil": 0.05515003204345703, "dL_bp": -0.2579076290130615, "ratio": -0.2138363733345166}}, {"step": 100, "epoch": 0, "train_loss": 4.048423767089844, "cos_r_negg": [-0.00038708001375198364, 0.012312850914895535, -0.18562011420726776], "cos_innovation_negg": [0.0059972964227199554, 0.3373491168022156, -0.40158626437187195], "cos_apical_negg": [-0.00038708001375198364, 0.012312850914895535, -0.18562011420726776], "cos_Ac_negg": [0.0059818970039486885, 0.337344229221344, -0.4015938341617584], "r_norm": [10.49108600616455, 9.80578327178955, 9.477056503295898]}, {"step": 200, "epoch": 0, "train_loss": 7.680235385894775, "cos_r_negg": [-0.013611581176519394, -0.10219574719667435, -0.11266361176967621], "cos_innovation_negg": [-0.0046990178525447845, -0.2277299463748932, -0.011543679982423782], "cos_apical_negg": [-0.013611581176519394, -0.10219574719667435, -0.11266361176967621], "cos_Ac_negg": [-0.004698587581515312, -0.22771966457366943, -0.01154409721493721], "r_norm": [10.431875228881836, 9.767824172973633, 9.762587547302246]}, {"step": 300, "epoch": 0, "train_loss": 13.205154418945312, "cos_r_negg": [0.005946682766079903, 0.07814735919237137, -0.223262757062912], "cos_innovation_negg": [0.014742700383067131, 0.014042189344763756, -0.17553220689296722], "cos_apical_negg": [0.005946682766079903, 0.07814735919237137, -0.223262757062912], "cos_Ac_negg": [0.01473372895270586, 0.0139680290594697, -0.1755208969116211], "r_norm": [10.227781295776367, 9.750490188598633, 9.6602144241333]}, {"step": 400, "epoch": 0, "train_loss": 8.00300407409668, "cos_r_negg": [0.008192419074475765, -0.03756037354469299, -0.2376447468996048], "cos_innovation_negg": [-0.01783088967204094, -0.08315109461545944, -0.018611211329698563], "cos_apical_negg": [0.008192419074475765, -0.03756037354469299, -0.2376447468996048], "cos_Ac_negg": [-0.017820363864302635, -0.08314765244722366, -0.01861168071627617], "r_norm": [10.095707893371582, 9.557220458984375, 9.794044494628906]}, {"epoch_end": 0, "step": 469, "test_acc": 0.1028, "test_loss": 8.2363947265625}, {"step": 500, "epoch": 1, "train_loss": 8.892101287841797, "cos_r_negg": [0.019596759229898453, -0.04804927110671997, 0.12499937415122986], "cos_innovation_negg": [0.012671569362282753, -0.014474717900156975, 0.4192502498626709], "cos_apical_negg": [0.019596759229898453, -0.04804927110671997, 0.12499937415122986], "cos_Ac_negg": [0.012665145099163055, -0.014520451426506042, 0.4192420244216919], "r_norm": [9.692049026489258, 9.434730529785156, 9.41353988647461], "ldr": {"dL_sdil": -1.0833816528320312, "dL_bp": -1.0833816528320312, "ratio": 1.0}}, {"step": 600, "epoch": 1, "train_loss": 8.474376678466797, "cos_r_negg": [0.02553371898829937, 0.1724296510219574, -0.354047954082489], "cos_innovation_negg": [0.045577164739370346, 0.23265646398067474, -0.2001267671585083], "cos_apical_negg": [0.02553371898829937, 0.1724296510219574, -0.354047954082489], "cos_Ac_negg": [0.045568183064460754, 0.2326515167951584, -0.20013195276260376], "r_norm": [9.443550109863281, 9.615306854248047, 9.66866397857666]}, {"step": 700, "epoch": 1, "train_loss": 5.93480920791626, "cos_r_negg": [0.03728172928094864, -0.10968152433633804, -0.1812736690044403], "cos_innovation_negg": [0.017669204622507095, 0.38141557574272156, -0.20808491110801697], "cos_apical_negg": [0.03728172928094864, -0.10968152433633804, -0.1812736690044403], "cos_Ac_negg": [0.017658857628703117, 0.38139232993125916, -0.208084836602211], "r_norm": [8.878643989562988, 8.826092720031738, 8.838126182556152]}, {"step": 800, "epoch": 1, "train_loss": 13.359890937805176, "cos_r_negg": [0.02385527268052101, -0.07425002753734589, 0.07332029938697815], "cos_innovation_negg": [-0.027477825060486794, 0.2613312602043152, 0.14789097011089325], "cos_apical_negg": [0.02385527268052101, -0.07425002753734589, 0.07332029938697815], "cos_Ac_negg": [-0.027491716668009758, 0.261358380317688, 0.14788001775741577], "r_norm": [8.751726150512695, 8.799842834472656, 8.603059768676758]}, {"step": 900, "epoch": 1, "train_loss": 19.79599380493164, "cos_r_negg": [-0.015940885990858078, 0.1013120710849762, -0.25914034247398376], "cos_innovation_negg": [0.018351517617702484, 0.23802566528320312, -0.2530173659324646], "cos_apical_negg": [-0.015940885990858078, 0.1013120710849762, -0.25914034247398376], "cos_Ac_negg": [0.018346844241023064, 0.23782522976398468, -0.25302356481552124], "r_norm": [8.50736141204834, 8.42558479309082, 8.270204544067383]}, {"epoch_end": 1, "step": 938, "test_acc": 0.0958, "test_loss": 6.2221748046875}, {"step": 1000, "epoch": 2, "train_loss": 10.403169631958008, "cos_r_negg": [-0.016361689195036888, 0.09664098173379898, 0.07940943539142609], "cos_innovation_negg": [-0.004043289925903082, -0.1956271231174469, -0.08518767356872559], "cos_apical_negg": [-0.016361689195036888, 0.09664098173379898, 0.07940943539142609], "cos_Ac_negg": [-0.004057942423969507, -0.19564276933670044, -0.08519452065229416], "r_norm": [8.528695106506348, 8.44913101196289, 8.408604621887207], "ldr": {"dL_sdil": -1.2459030151367188, "dL_bp": -1.2459030151367188, "ratio": 1.0}}, {"step": 1100, "epoch": 2, "train_loss": 13.219457626342773, "cos_r_negg": [-0.007558685727417469, 0.20877350866794586, -0.37985098361968994], "cos_innovation_negg": [-0.0724012553691864, 0.1799122393131256, 0.23801423609256744], "cos_apical_negg": [-0.007558685727417469, 0.20877350866794586, -0.37985098361968994], "cos_Ac_negg": [-0.0724102035164833, 0.17991892993450165, 0.23813165724277496], "r_norm": [8.575762748718262, 8.789449691772461, 8.783294677734375]}, {"step": 1200, "epoch": 2, "train_loss": 12.265817642211914, "cos_r_negg": [-0.008332806639373302, -0.09049943834543228, 0.31531596183776855], "cos_innovation_negg": [0.02332516573369503, -0.032542645931243896, 0.12632256746292114], "cos_apical_negg": [-0.008332806639373302, -0.09049943834543228, 0.31531596183776855], "cos_Ac_negg": [0.0233245100826025, -0.03225938603281975, 0.12631994485855103], "r_norm": [8.28215217590332, 8.727643966674805, 8.610010147094727]}, {"step": 1300, "epoch": 2, "train_loss": 12.69871711730957, "cos_r_negg": [-0.033486101776361465, 0.09673546999692917, -0.09153277426958084], "cos_innovation_negg": [-0.0815923884510994, -0.23683254420757294, 0.029484009370207787], "cos_apical_negg": [-0.033486101776361465, 0.09673546999692917, -0.09153277426958084], "cos_Ac_negg": [-0.08158624172210693, -0.23682795464992523, 0.02948380447924137], "r_norm": [8.567813873291016, 8.611093521118164, 8.656942367553711]}, {"step": 1400, "epoch": 2, "train_loss": 9.126750946044922, "cos_r_negg": [-0.0028847698122262955, 0.038713324815034866, -0.10155009478330612], "cos_innovation_negg": [-0.04885728657245636, 0.09896579384803772, -0.05821666121482849], "cos_apical_negg": [-0.0028847698122262955, 0.038713324815034866, -0.10155009478330612], "cos_Ac_negg": [-0.04884689301252365, 0.09898638725280762, -0.05821705609560013], "r_norm": [8.293462753295898, 8.819238662719727, 8.696137428283691]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.0982, "test_loss": 9.417101171875}, {"step": 1500, "epoch": 3, "train_loss": 10.652914047241211, "cos_r_negg": [-0.02233140543103218, 0.028967654332518578, -0.10044126212596893], "cos_innovation_negg": [-0.07973453402519226, -0.12923435866832733, -0.17853903770446777], "cos_apical_negg": [-0.02233140543103218, 0.028967654332518578, -0.10044126212596893], "cos_Ac_negg": [-0.07974031567573547, -0.12923023104667664, -0.17853903770446777], "r_norm": [8.33251953125, 8.236413955688477, 8.087556838989258], "ldr": {"dL_sdil": -4.648153305053711, "dL_bp": -4.648153305053711, "ratio": 1.0}}, {"step": 1600, "epoch": 3, "train_loss": 12.458199501037598, "cos_r_negg": [0.00986754335463047, -0.11913810670375824, -0.0482497364282608], "cos_innovation_negg": [0.03929536044597626, -0.22934722900390625, 0.584274411201477], "cos_apical_negg": [0.00986754335463047, -0.11913810670375824, -0.0482497364282608], "cos_Ac_negg": [0.039303041994571686, -0.22934341430664062, 0.584274411201477], "r_norm": [8.355783462524414, 8.55724811553955, 8.539680480957031]}, {"step": 1700, "epoch": 3, "train_loss": 13.3245210647583, "cos_r_negg": [0.0343148335814476, -0.21725930273532867, -0.565794825553894], "cos_innovation_negg": [-0.015311628580093384, 0.1923694610595703, 0.7810370326042175], "cos_apical_negg": [0.0343148335814476, -0.21725930273532867, -0.565794825553894], "cos_Ac_negg": [-0.0153830386698246, 0.19240477681159973, 0.7810370326042175], "r_norm": [8.39525032043457, 8.216879844665527, 8.287924766540527]}, {"step": 1800, "epoch": 3, "train_loss": 11.977592468261719, "cos_r_negg": [-0.01665971614420414, -0.10355214774608612, -0.296875], "cos_innovation_negg": [-0.013934001326560974, 0.04588707536458969, -0.37890625], "cos_apical_negg": [-0.01665971614420414, -0.10355214774608612, -0.296875], "cos_Ac_negg": [-0.013435456901788712, 0.04583577811717987, -0.37890625], "r_norm": [8.463891983032227, 8.802080154418945, 8.727274894714355]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.098, "test_loss": 11.054625}, {"step": 1900, "epoch": 4, "train_loss": 13.151737213134766, "cos_r_negg": [0.0704783946275711, -0.5064337253570557, -0.78125], "cos_innovation_negg": [0.06315523386001587, 0.27298060059547424, 0.78515625], "cos_apical_negg": [0.0704783946275711, -0.5064337253570557, -0.78125], "cos_Ac_negg": [0.0631660521030426, 0.2729974389076233, 0.78515625], "r_norm": [7.965924263000488, 7.917893409729004, 7.962090492248535]}, {"step": 2000, "epoch": 4, "train_loss": 15.741877555847168, "cos_r_negg": [-0.043511971831321716, -0.6436718702316284, -0.78125], "cos_innovation_negg": [0.16512197256088257, -0.10344197601079941, 0.62109375], "cos_apical_negg": [-0.043511971831321716, -0.6436718702316284, -0.78125], "cos_Ac_negg": [0.16510631144046783, -0.10340915620326996, 0.62109375], "r_norm": [8.160762786865234, 8.159454345703125, 8.06200885772705], "ldr": {"dL_sdil": -2.320943832397461, "dL_bp": -2.3209447860717773, "ratio": 0.999999589100817}}, {"step": 2100, "epoch": 4, "train_loss": 10.681032180786133, "cos_r_negg": [-0.0950709879398346, 0.17797280848026276, 0.2734375], "cos_innovation_negg": [0.12455791234970093, 0.088068887591362, 0.56640625], "cos_apical_negg": [-0.0950709879398346, 0.17797280848026276, 0.2734375], "cos_Ac_negg": [0.12455766648054123, 0.08806349337100983, 0.56640625], "r_norm": [8.547739028930664, 8.701484680175781, 8.717161178588867]}, {"step": 2200, "epoch": 4, "train_loss": 14.67729377746582, "cos_r_negg": [-0.024390969425439835, -0.0477474108338356, -0.1015625], "cos_innovation_negg": [0.030237952247262, 0.17822256684303284, 0.0859375], "cos_apical_negg": [-0.024390969425439835, -0.0477474108338356, -0.1015625], "cos_Ac_negg": [0.030224112793803215, 0.17822477221488953, 0.0859375], "r_norm": [8.491247177124023, 8.701871871948242, 8.658407211303711]}, {"step": 2300, "epoch": 4, "train_loss": 12.383636474609375, "cos_r_negg": [0.07453485578298569, 0.00718866940587759, 0.0546875], "cos_innovation_negg": [-0.10117971897125244, -0.056033454835414886, 0.21484375], "cos_apical_negg": [0.07453485578298569, 0.00718866940587759, 0.0546875], "cos_Ac_negg": [-0.10116216540336609, -0.05603320524096489, 0.21484375], "r_norm": [7.9048686027526855, 8.120739936828613, 8.041391372680664]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.098, "test_loss": 7.797568017578125}, {"step": 2400, "epoch": 5, "train_loss": 9.360052108764648, "cos_r_negg": [0.05859808251261711, 0.031209755688905716, 0.0546875], "cos_innovation_negg": [0.02489345334470272, -0.07161745429039001, 0.15234375], "cos_apical_negg": [0.05859808251261711, 0.031209755688905716, 0.0546875], "cos_Ac_negg": [0.024872617796063423, -0.0716203972697258, 0.15234375], "r_norm": [8.386375427246094, 8.399620056152344, 8.3935546875]}, {"step": 2500, "epoch": 5, "train_loss": 14.313504219055176, "cos_r_negg": [0.14963398873806, 0.07903169840574265, 0.21875], "cos_innovation_negg": [0.10916762053966522, -0.06470189988613129, 0.421875], "cos_apical_negg": [0.14963398873806, 0.07903169840574265, 0.21875], "cos_Ac_negg": [0.1091456189751625, -0.0647021010518074, 0.421875], "r_norm": [9.090614318847656, 9.206798553466797, 9.406774520874023], "ldr": {"dL_sdil": -1.5052204132080078, "dL_bp": -1.5052204132080078, "ratio": 1.0}}, {"step": 2600, "epoch": 5, "train_loss": 16.195253372192383, "cos_r_negg": [-0.06374979764223099, -0.32187503576278687, -0.56640625], "cos_innovation_negg": [0.30853092670440674, 0.4852294921875, 1.0], "cos_apical_negg": [-0.06374979764223099, -0.32187503576278687, -0.56640625], "cos_Ac_negg": [0.3085094392299652, 0.4852292239665985, 1.0], "r_norm": [7.5411272048950195, 7.458311080932617, 7.411427974700928]}, {"step": 2700, "epoch": 5, "train_loss": 13.607898712158203, "cos_r_negg": [0.1364213526248932, -0.059440236538648605, -0.1015625], "cos_innovation_negg": [-0.06791872531175613, 0.014796055853366852, 0.53515625], "cos_apical_negg": [0.1364213526248932, -0.059440236538648605, -0.1015625], "cos_Ac_negg": [-0.06784215569496155, 0.014754671603441238, 0.53515625], "r_norm": [8.212850570678711, 8.385347366333008, 8.360902786254883]}, {"step": 2800, "epoch": 5, "train_loss": 12.251354217529297, "cos_r_negg": [0.05929980054497719, 0.02887525036931038, 0.0546875], "cos_innovation_negg": [-0.10293389111757278, -0.09590262174606323, 0.16015625], "cos_apical_negg": [0.05929980054497719, 0.02887525036931038, 0.0546875], "cos_Ac_negg": [-0.1026286780834198, -0.09592217952013016, 0.16015625], "r_norm": [8.372848510742188, 8.53820514678955, 8.538537979125977]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.1009, "test_loss": 9.24760322265625}, {"step": 2900, "epoch": 6, "train_loss": 15.240873336791992, "cos_r_negg": [-0.00856858491897583, -0.1104426234960556, -0.13671875], "cos_innovation_negg": [0.226836159825325, -0.138884037733078, 0.78515625], "cos_apical_negg": [-0.00856858491897583, -0.1104426234960556, -0.13671875], "cos_Ac_negg": [0.2267845869064331, -0.1388828307390213, 0.78515625], "r_norm": [8.276073455810547, 8.262262344360352, 8.26427173614502]}, {"step": 3000, "epoch": 6, "train_loss": 13.45813274383545, "cos_r_negg": [-0.2407328486442566, -0.4382801651954651, -0.78125], "cos_innovation_negg": [-0.38163360953330994, -0.16624566912651062, 0.359375], "cos_apical_negg": [-0.2407328486442566, -0.4382801651954651, -0.78125], "cos_Ac_negg": [-0.3816559314727783, -0.16624721884727478, 0.359375], "r_norm": [9.034111976623535, 9.004579544067383, 8.994977951049805], "ldr": {"dL_sdil": -3.0672531127929688, "dL_bp": -3.0672531127929688, "ratio": 1.0}}, {"step": 3100, "epoch": 6, "train_loss": 10.449892044067383, "cos_r_negg": [-0.40032556653022766, -0.4496147632598877, -0.78125], "cos_innovation_negg": [0.04339674860239029, 0.2506718039512634, 1.0], "cos_apical_negg": [-0.40032556653022766, -0.4496147632598877, -0.78125], "cos_Ac_negg": [0.04338822141289711, 0.2506703734397888, 1.0], "r_norm": [8.184335708618164, 8.155200958251953, 8.128531455993652]}, {"step": 3200, "epoch": 6, "train_loss": 11.123907089233398, "cos_r_negg": [-0.27482232451438904, -0.19212891161441803, -0.296875], "cos_innovation_negg": [-0.0446096733212471, -0.03836378827691078, 0.73046875], "cos_apical_negg": [-0.27482232451438904, -0.19212891161441803, -0.296875], "cos_Ac_negg": [-0.04462036117911339, -0.03836483135819435, 0.73046875], "r_norm": [8.158368110656738, 8.149415016174316, 8.115013122558594]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.1032, "test_loss": 17.598535546875}, {"step": 3300, "epoch": 7, "train_loss": 6.77842903137207, "cos_r_negg": [0.03178637847304344, 0.02861172705888748, 0.0546875], "cos_innovation_negg": [-0.17581456899642944, -0.09045815467834473, 0.6171875], "cos_apical_negg": [0.03178637847304344, 0.02861172705888748, 0.0546875], "cos_Ac_negg": [-0.1761072427034378, -0.09046730399131775, 0.6171875], "r_norm": [8.44297981262207, 8.518793106079102, 8.53906536102295]}, {"step": 3400, "epoch": 7, "train_loss": 10.882484436035156, "cos_r_negg": [-0.3764663636684418, -0.4599994719028473, -0.78125], "cos_innovation_negg": [0.471699595451355, 0.5242045521736145, 1.0], "cos_apical_negg": [-0.3764663636684418, -0.4599994719028473, -0.78125], "cos_Ac_negg": [0.47167664766311646, 0.524203896522522, 1.0], "r_norm": [7.732322692871094, 7.713907241821289, 7.694057941436768]}, {"step": 3500, "epoch": 7, "train_loss": 19.509206771850586, "cos_r_negg": [-0.051472343504428864, -0.18815027177333832, -0.296875], "cos_innovation_negg": [-0.23581969738006592, -0.24651487171649933, 0.5859375], "cos_apical_negg": [-0.051472343504428864, -0.18815027177333832, -0.296875], "cos_Ac_negg": [-0.23586036264896393, -0.24651530385017395, 0.5859375], "r_norm": [8.643613815307617, 8.701316833496094, 8.70458984375], "ldr": {"dL_sdil": -1.2038860321044922, "dL_bp": -1.2038860321044922, "ratio": 1.0}}, {"step": 3600, "epoch": 7, "train_loss": 10.769990921020508, "cos_r_negg": [-0.4085528552532196, -0.451077401638031, -0.78125], "cos_innovation_negg": [0.5861038565635681, 0.5647537708282471, 1.0], "cos_apical_negg": [-0.4085528552532196, -0.451077401638031, -0.78125], "cos_Ac_negg": [0.5860505104064941, 0.5647536516189575, 1.0], "r_norm": [7.625499725341797, 7.612261772155762, 7.5605998039245605]}, {"step": 3700, "epoch": 7, "train_loss": 12.681997299194336, "cos_r_negg": [-0.08789121359586716, -0.3333163261413574, -0.51171875], "cos_innovation_negg": [-0.16150905191898346, -0.4525165557861328, 0.37109375], "cos_apical_negg": [-0.08789121359586716, -0.3333163261413574, -0.51171875], "cos_Ac_negg": [-0.16150511801242828, -0.45251673460006714, 0.37109375], "r_norm": [8.740745544433594, 8.72075366973877, 8.748673439025879]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.0974, "test_loss": 12.719573046875}, {"step": 3800, "epoch": 8, "train_loss": 7.220987319946289, "cos_r_negg": [0.10857321321964264, 0.06563562154769897, 0.1171875], "cos_innovation_negg": [-0.09899701923131943, -0.010265226475894451, 0.84375], "cos_apical_negg": [0.10857321321964264, 0.06563562154769897, 0.1171875], "cos_Ac_negg": [-0.0989745706319809, -0.010265344753861427, 0.84375], "r_norm": [8.646788597106934, 8.717842102050781, 8.777027130126953]}, {"step": 3900, "epoch": 8, "train_loss": 18.84990119934082, "cos_r_negg": [-0.41477975249290466, -0.3563691973686218, -0.56640625], "cos_innovation_negg": [0.11122597008943558, 0.35404348373413086, 0.53515625], "cos_apical_negg": [-0.41477975249290466, -0.3563691973686218, -0.56640625], "cos_Ac_negg": [0.11118855327367783, 0.35404258966445923, 0.53515625], "r_norm": [8.358858108520508, 8.309804916381836, 8.362595558166504]}, {"step": 4000, "epoch": 8, "train_loss": 11.843727111816406, "cos_r_negg": [0.06424508988857269, -0.06611384451389313, -0.1015625], "cos_innovation_negg": [-0.26604729890823364, -0.16560904681682587, 0.2578125], "cos_apical_negg": [0.06424508988857269, -0.06611384451389313, -0.1015625], "cos_Ac_negg": [-0.2660340368747711, -0.16560426354408264, 0.2578125], "r_norm": [8.712605476379395, 8.75984001159668, 8.80296516418457], "ldr": {"dL_sdil": -5.210995674133301, "dL_bp": -5.210995674133301, "ratio": 1.0}}, {"step": 4100, "epoch": 8, "train_loss": 11.043825149536133, "cos_r_negg": [-0.012536229565739632, -0.19439512491226196, -0.296875], "cos_innovation_negg": [-0.2442316710948944, 0.044733524322509766, 0.8046875], "cos_apical_negg": [-0.012536229565739632, -0.19439512491226196, -0.296875], "cos_Ac_negg": [-0.24422474205493927, 0.044733546674251556, 0.8046875], "r_norm": [8.27298355102539, 8.28322982788086, 8.324411392211914]}, {"step": 4200, "epoch": 8, "train_loss": 10.813334465026855, "cos_r_negg": [-0.07172022014856339, -0.09402187168598175, -0.140625], "cos_innovation_negg": [-0.07185393571853638, -0.055279091000556946, 0.8046875], "cos_apical_negg": [-0.07172022014856339, -0.09402187168598175, -0.140625], "cos_Ac_negg": [-0.07181825488805771, -0.05527891218662262, 0.8046875], "r_norm": [8.569534301757812, 8.555490493774414, 8.607816696166992]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.0892, "test_loss": 11.2686205078125}, {"step": 4300, "epoch": 9, "train_loss": 11.30282974243164, "cos_r_negg": [-0.42207491397857666, -0.4704839587211609, -0.78125], "cos_innovation_negg": [0.6612135767936707, 0.6869399547576904, 1.0], "cos_apical_negg": [-0.42207491397857666, -0.4704839587211609, -0.78125], "cos_Ac_negg": [0.661078155040741, 0.6869426965713501, 1.0], "r_norm": [6.958515167236328, 6.957447528839111, 6.912569522857666]}, {"step": 4400, "epoch": 9, "train_loss": 12.547195434570312, "cos_r_negg": [-0.22351884841918945, -0.3272722363471985, -0.51171875], "cos_innovation_negg": [-0.438857764005661, -0.6343268752098083, 0.78515625], "cos_apical_negg": [-0.22351884841918945, -0.3272722363471985, -0.51171875], "cos_Ac_negg": [-0.4388698935508728, -0.6343271732330322, 0.78515625], "r_norm": [8.885265350341797, 8.9166259765625, 8.973077774047852]}, {"step": 4500, "epoch": 9, "train_loss": 13.633489608764648, "cos_r_negg": [-0.07313361018896103, -0.19560879468917847, -0.296875], "cos_innovation_negg": [-0.09623672813177109, -0.016946494579315186, 0.6484375], "cos_apical_negg": [-0.07313361018896103, -0.19560879468917847, -0.296875], "cos_Ac_negg": [-0.0962434709072113, -0.01694667339324951, 0.6484375], "r_norm": [8.246091842651367, 8.248424530029297, 8.283790588378906], "ldr": {"dL_sdil": -1.6996498107910156, "dL_bp": -1.6996498107910156, "ratio": 1.0}}, {"step": 4600, "epoch": 9, "train_loss": 11.515636444091797, "cos_r_negg": [-0.4379178583621979, -0.476169615983963, -0.78125], "cos_innovation_negg": [0.507658839225769, 0.6764594316482544, 1.0], "cos_apical_negg": [-0.4379178583621979, -0.476169615983963, -0.78125], "cos_Ac_negg": [0.5076534152030945, 0.6764594912528992, 1.0], "r_norm": [7.644908905029297, 7.651847839355469, 7.635805606842041]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.1032, "test_loss": 12.22563271484375}, {"step": 4700, "epoch": 10, "train_loss": 11.154718399047852, "cos_r_negg": [0.04211030900478363, -0.051484204828739166, -0.078125], "cos_innovation_negg": [0.17123736441135406, 0.021926064044237137, 0.484375], "cos_apical_negg": [0.04211030900478363, -0.051484204828739166, -0.078125], "cos_Ac_negg": [0.17122957110404968, 0.02192624658346176, 0.484375], "r_norm": [8.554386138916016, 8.57331371307373, 8.582290649414062]}, {"step": 4800, "epoch": 10, "train_loss": 10.413039207458496, "cos_r_negg": [-0.06587191671133041, -0.1854400783777237, -0.296875], "cos_innovation_negg": [0.33011189103126526, 0.09548874199390411, 0.8046875], "cos_apical_negg": [-0.06587191671133041, -0.1854400783777237, -0.296875], "cos_Ac_negg": [0.32983729243278503, 0.09548848122358322, 0.8046875], "r_norm": [8.536161422729492, 8.55754280090332, 8.540809631347656]}, {"step": 4900, "epoch": 10, "train_loss": 8.770610809326172, "cos_r_negg": [-0.16251826286315918, -0.3209840953350067, -0.51171875], "cos_innovation_negg": [0.286214679479599, -0.12868893146514893, 0.78515625], "cos_apical_negg": [-0.16251826286315918, -0.3209840953350067, -0.51171875], "cos_Ac_negg": [0.2861725389957428, -0.12868985533714294, 0.78515625], "r_norm": [8.486433029174805, 8.503961563110352, 8.495394706726074]}, {"step": 5000, "epoch": 10, "train_loss": 10.889909744262695, "cos_r_negg": [0.0900082215666771, -0.08585497736930847, -0.140625], "cos_innovation_negg": [-0.015633879229426384, -0.07281642407178879, 0.421875], "cos_apical_negg": [0.0900082215666771, -0.08585497736930847, -0.140625], "cos_Ac_negg": [-0.015591621398925781, -0.07281635701656342, 0.421875], "r_norm": [8.31772518157959, 8.335901260375977, 8.394617080688477], "ldr": {"dL_sdil": -2.2710018157958984, "dL_bp": -2.2710018157958984, "ratio": 1.0}}, {"step": 5100, "epoch": 10, "train_loss": 14.994162559509277, "cos_r_negg": [0.10927048325538635, -0.06370556354522705, -0.09765625], "cos_innovation_negg": [0.3110046982765198, -0.038481324911117554, 0.46484375], "cos_apical_negg": [0.10927048325538635, -0.06370556354522705, -0.09765625], "cos_Ac_negg": [0.31100744009017944, -0.03848128020763397, 0.46484375], "r_norm": [8.779733657836914, 8.800010681152344, 8.825196266174316]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.0974, "test_loss": 13.11488681640625}, {"step": 5200, "epoch": 11, "train_loss": 14.59203052520752, "cos_r_negg": [-0.37050753831863403, -0.23410217463970184, -0.37109375], "cos_innovation_negg": [0.026665732264518738, -0.012106293812394142, 0.73046875], "cos_apical_negg": [-0.37050753831863403, -0.23410217463970184, -0.37109375], "cos_Ac_negg": [0.026613960042595863, -0.012103769928216934, 0.73046875], "r_norm": [8.561629295349121, 8.571050643920898, 8.591808319091797]}, {"step": 5300, "epoch": 11, "train_loss": 13.225076675415039, "cos_r_negg": [0.10546252131462097, -0.06255826354026794, -0.1015625], "cos_innovation_negg": [-0.15438078343868256, -0.13905379176139832, 0.4609375], "cos_apical_negg": [0.10546252131462097, -0.06255826354026794, -0.1015625], "cos_Ac_negg": [-0.15430378913879395, -0.13905397057533264, 0.4609375], "r_norm": [8.374811172485352, 8.399364471435547, 8.391115188598633]}, {"step": 5400, "epoch": 11, "train_loss": 15.979743957519531, "cos_r_negg": [0.19339673221111298, 0.13396701216697693, 0.21875], "cos_innovation_negg": [0.03953156992793083, 0.01834408938884735, 0.78125], "cos_apical_negg": [0.19339673221111298, 0.13396701216697693, 0.21875], "cos_Ac_negg": [0.039623573422431946, 0.01834414154291153, 0.78125], "r_norm": [8.356511116027832, 8.387165069580078, 8.43072509765625]}, {"step": 5500, "epoch": 11, "train_loss": 17.079755783081055, "cos_r_negg": [-0.17131507396697998, -0.12880662083625793, -0.20703125], "cos_innovation_negg": [0.06365390121936798, 0.13643576204776764, 0.35546875], "cos_apical_negg": [-0.17131507396697998, -0.12880662083625793, -0.20703125], "cos_Ac_negg": [0.06359851360321045, 0.13643546402454376, 0.35546875], "r_norm": [8.2498779296875, 8.280290603637695, 8.350334167480469], "ldr": {"dL_sdil": -1.9855546951293945, "dL_bp": -1.9855546951293945, "ratio": 1.0}}, {"step": 5600, "epoch": 11, "train_loss": 13.779155731201172, "cos_r_negg": [0.05128047987818718, -0.19811663031578064, -0.31640625], "cos_innovation_negg": [0.09084538370370865, -0.20866376161575317, 0.04296875], "cos_apical_negg": [0.05128047987818718, -0.19811663031578064, -0.31640625], "cos_Ac_negg": [0.09087378531694412, -0.20866474509239197, 0.04296875], "r_norm": [8.729887008666992, 8.74886703491211, 8.835797309875488]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.0974, "test_loss": 7.890156201171875}, {"step": 5700, "epoch": 12, "train_loss": 13.740019798278809, "cos_r_negg": [-0.022249044850468636, -0.19086125493049622, -0.296875], "cos_innovation_negg": [0.25283122062683105, 0.1070123165845871, 0.6484375], "cos_apical_negg": [-0.022249044850468636, -0.19086125493049622, -0.296875], "cos_Ac_negg": [0.2528183162212372, 0.1070125475525856, 0.6484375], "r_norm": [8.25234317779541, 8.273443222045898, 8.24481201171875]}, {"step": 5800, "epoch": 12, "train_loss": 15.185934066772461, "cos_r_negg": [-0.47118210792541504, -0.49360305070877075, -0.78515625], "cos_innovation_negg": [-0.042259059846401215, 0.5175637602806091, 0.51171875], "cos_apical_negg": [-0.47118210792541504, -0.49360305070877075, -0.78515625], "cos_Ac_negg": [-0.0423424057662487, 0.5180661678314209, 0.51171875], "r_norm": [8.352450370788574, 8.361159324645996, 8.426286697387695]}, {"step": 5900, "epoch": 12, "train_loss": 8.332021713256836, "cos_r_negg": [-0.012060514651238918, -0.06310335546731949, -0.1015625], "cos_innovation_negg": [-0.050032176077365875, -0.0006044637411832809, 1.0], "cos_apical_negg": [-0.012060514651238918, -0.06310335546731949, -0.1015625], "cos_Ac_negg": [-0.0500577911734581, -0.0006053010001778603, 1.0], "r_norm": [7.918285369873047, 7.925766468048096, 7.932440280914307]}, {"step": 6000, "epoch": 12, "train_loss": 12.282862663269043, "cos_r_negg": [-0.13919560611248016, -0.3347434401512146, -0.515625], "cos_innovation_negg": [-0.0738874301314354, -0.056763142347335815, 0.78125], "cos_apical_negg": [-0.13919560611248016, -0.3347434401512146, -0.515625], "cos_Ac_negg": [-0.07386118173599243, -0.056763119995594025, 0.78125], "r_norm": [8.246475219726562, 8.265886306762695, 8.238741874694824], "ldr": {"dL_sdil": -1.0019464492797852, "dL_bp": -1.0019464492797852, "ratio": 1.0}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.101, "test_loss": 16.701590234375}, {"step": 6100, "epoch": 13, "train_loss": 13.568330764770508, "cos_r_negg": [-0.3286958932876587, -0.23590868711471558, -0.37109375], "cos_innovation_negg": [-0.5864284038543701, -0.24276292324066162, 0.35546875], "cos_apical_negg": [-0.3286958932876587, -0.23590868711471558, -0.37109375], "cos_Ac_negg": [-0.5864269733428955, -0.24276292324066162, 0.35546875], "r_norm": [8.762467384338379, 8.775313377380371, 8.811768531799316]}, {"step": 6200, "epoch": 13, "train_loss": 6.621945381164551, "cos_r_negg": [-0.12752246856689453, -0.3257513642311096, -0.51171875], "cos_innovation_negg": [0.04779680445790291, -0.014845471829175949, 1.0], "cos_apical_negg": [-0.12752246856689453, -0.3257513642311096, -0.51171875], "cos_Ac_negg": [0.04783311486244202, -0.014845405705273151, 1.0], "r_norm": [8.24631118774414, 8.268777847290039, 8.265740394592285]}, {"step": 6300, "epoch": 13, "train_loss": 5.512476921081543, "cos_r_negg": [0.1348036825656891, 0.035266757011413574, 0.0546875], "cos_innovation_negg": [-0.2639142870903015, -0.3019446134567261, 0.6171875], "cos_apical_negg": [0.1348036825656891, 0.035266757011413574, 0.0546875], "cos_Ac_negg": [-0.2638952434062958, -0.3019335865974426, 0.6171875], "r_norm": [7.887872695922852, 7.902234077453613, 7.9197540283203125]}, {"step": 6400, "epoch": 13, "train_loss": 13.546648025512695, "cos_r_negg": [0.011138486675918102, -0.09149190783500671, -0.140625], "cos_innovation_negg": [-0.11614083498716354, -0.14522939920425415, 0.3984375], "cos_apical_negg": [0.011138486675918102, -0.09149190783500671, -0.140625], "cos_Ac_negg": [-0.11569877713918686, -0.14523273706436157, 0.3984375], "r_norm": [7.982773780822754, 8.00344181060791, 8.06374740600586]}, {"step": 6500, "epoch": 13, "train_loss": 11.577411651611328, "cos_r_negg": [0.029955074191093445, -0.06374683976173401, -0.1015625], "cos_innovation_negg": [-0.1661345660686493, -0.09280157089233398, 0.84375], "cos_apical_negg": [0.029955074191093445, -0.06374683976173401, -0.1015625], "cos_Ac_negg": [-0.16615183651447296, -0.09280161559581757, 0.84375], "r_norm": [8.310872077941895, 8.319803237915039, 8.357292175292969], "ldr": {"dL_sdil": -1.6535072326660156, "dL_bp": -1.6535072326660156, "ratio": 1.0}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.0892, "test_loss": 11.9782271484375}, {"step": 6600, "epoch": 14, "train_loss": 8.526567459106445, "cos_r_negg": [-0.3817954957485199, -0.5017862319946289, -0.78125], "cos_innovation_negg": [-0.34609857201576233, -0.09906160831451416, 0.78515625], "cos_apical_negg": [-0.3817954957485199, -0.5017862319946289, -0.78125], "cos_Ac_negg": [-0.3461563289165497, -0.09906161576509476, 0.78515625], "r_norm": [8.510602951049805, 8.5277099609375, 8.542966842651367]}, {"step": 6700, "epoch": 14, "train_loss": 8.28334903717041, "cos_r_negg": [-0.4204058051109314, -0.5008850693702698, -0.78125], "cos_innovation_negg": [-0.15817169845104218, -0.027469102293252945, 1.0], "cos_apical_negg": [-0.4204058051109314, -0.5008850693702698, -0.78125], "cos_Ac_negg": [-0.158253014087677, -0.027468973770737648, 1.0], "r_norm": [8.390499114990234, 8.408519744873047, 8.4287109375]}, {"step": 6800, "epoch": 14, "train_loss": 13.922309875488281, "cos_r_negg": [-0.3424050807952881, -0.5002288818359375, -0.78125], "cos_innovation_negg": [0.04831494763493538, 0.06858976930379868, 1.0], "cos_apical_negg": [-0.3424050807952881, -0.5002288818359375, -0.78125], "cos_Ac_negg": [0.046754706650972366, 0.06858483701944351, 1.0], "r_norm": [8.305824279785156, 8.321075439453125, 8.361061096191406]}, {"step": 6900, "epoch": 14, "train_loss": 8.011695861816406, "cos_r_negg": [-0.006971176713705063, -0.06244221329689026, -0.09765625], "cos_innovation_negg": [-0.053059712052345276, -0.14909477531909943, 0.78515625], "cos_apical_negg": [-0.006971176713705063, -0.06244221329689026, -0.09765625], "cos_Ac_negg": [-0.04014761000871658, -0.1490975022315979, 0.78515625], "r_norm": [8.24666976928711, 8.25935173034668, 8.308448791503906]}, {"step": 7000, "epoch": 14, "train_loss": 15.180927276611328, "cos_r_negg": [-0.08431568741798401, -0.04824742674827576, -0.078125], "cos_innovation_negg": [-0.08298332989215851, 0.039350561797618866, 0.8046875], "cos_apical_negg": [-0.08431568741798401, -0.04824742674827576, -0.078125], "cos_Ac_negg": [-0.08304154872894287, 0.03935077786445618, 0.8046875], "r_norm": [8.209514617919922, 8.224637985229492, 8.267744064331055], "ldr": {"dL_sdil": -2.422679901123047, "dL_bp": -2.422679901123047, "ratio": 1.0}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.1009, "test_loss": 20.922865625}], "final": {"test_acc": 0.1009, "test_loss": 20.922865625, "wall_s": 22.756576538085938, "cos_r_negg": [-0.421191930770874, -0.35951855778694153, -0.56640625], "cos_innovation_negg": [0.43286922574043274, 0.46319523453712463, 1.0], "cos_apical_negg": [-0.421191930770874, -0.35951855778694153, -0.56640625], "cos_Ac_negg": [0.4328594207763672, 0.46317994594573975, 1.0]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_residual_s0.json b/results/nuis_ctrl_v1_rho0p5_residual_s0.json new file mode 100644 index 0000000..b28c8b4 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_residual_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_residual_s0", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.3603768348693848, "cos_r_negg": [-0.019524920731782913, -0.025532349944114685, 0.033747125416994095], "cos_innovation_negg": [-0.019524920731782913, -0.025532349944114685, 0.033747125416994095], "cos_apical_negg": [-0.01679173856973648, -0.02367609366774559, 0.02439393848180771], "cos_Ac_negg": [-0.019524451345205307, -0.02553233876824379, 0.03374704346060753], "r_norm": [4.89827823638916, 4.826774597167969, 4.823909759521484], "ldr": {"dL_sdil": -0.00920414924621582, "dL_bp": -0.24071121215820312, "ratio": 0.03823731002678246}}, {"step": 100, "epoch": 0, "train_loss": 0.20806199312210083, "cos_r_negg": [0.45580482482910156, 0.37684938311576843, 0.3593536615371704], "cos_innovation_negg": [0.45580482482910156, 0.37684938311576843, 0.3593536615371704], "cos_apical_negg": [0.07743188738822937, 0.08167853951454163, 0.13730955123901367], "cos_Ac_negg": [0.45889896154403687, 0.38018500804901123, 0.361653596162796], "r_norm": [0.8243230581283569, 0.8080554604530334, 0.8057764768600464]}, {"step": 200, "epoch": 0, "train_loss": 0.32909178733825684, "cos_r_negg": [0.4774864912033081, 0.40109360218048096, 0.38381052017211914], "cos_innovation_negg": [0.4774864912033081, 0.40109360218048096, 0.38381052017211914], "cos_apical_negg": [0.06533575057983398, 0.08173279464244843, 0.12964117527008057], "cos_Ac_negg": [0.48442214727401733, 0.40823638439178467, 0.39088404178619385], "r_norm": [0.743273138999939, 0.7259811162948608, 0.7249500155448914]}, {"step": 300, "epoch": 0, "train_loss": 0.17777316272258759, "cos_r_negg": [0.48456960916519165, 0.4558779001235962, 0.42146602272987366], "cos_innovation_negg": [0.48456960916519165, 0.4558779001235962, 0.42146602272987366], "cos_apical_negg": [0.05611689016222954, 0.06970221549272537, 0.11532211303710938], "cos_Ac_negg": [0.4899787902832031, 0.45894038677215576, 0.42386552691459656], "r_norm": [0.5654057264328003, 0.5556600093841553, 0.5548664331436157]}, {"step": 400, "epoch": 0, "train_loss": 0.16122844815254211, "cos_r_negg": [0.5220173597335815, 0.48386672139167786, 0.439599871635437], "cos_innovation_negg": [0.5220173597335815, 0.48386672139167786, 0.439599871635437], "cos_apical_negg": [0.060485437512397766, 0.07159489393234253, 0.11227110773324966], "cos_Ac_negg": [0.5238868594169617, 0.48481935262680054, 0.43951553106307983], "r_norm": [0.5609845519065857, 0.5467572212219238, 0.5457912683486938]}, {"epoch_end": 0, "step": 469, "test_acc": 0.947, "test_loss": 0.18223495864868164}, {"step": 500, "epoch": 1, "train_loss": 0.24727237224578857, "cos_r_negg": [0.5016535520553589, 0.48832446336746216, 0.46044179797172546], "cos_innovation_negg": [0.5016535520553589, 0.48832446336746216, 0.46044179797172546], "cos_apical_negg": [0.05814739316701889, 0.0640498548746109, 0.1180124431848526], "cos_Ac_negg": [0.5116339325904846, 0.4933936893939972, 0.4625237286090851], "r_norm": [0.5076783895492554, 0.4894869327545166, 0.49166637659072876], "ldr": {"dL_sdil": -0.01459231972694397, "dL_bp": -0.01246914267539978, "ratio": 1.1702745013683242}}, {"step": 600, "epoch": 1, "train_loss": 0.09304194897413254, "cos_r_negg": [0.4987279176712036, 0.4869357645511627, 0.45599937438964844], "cos_innovation_negg": [0.4987279176712036, 0.4869357645511627, 0.45599937438964844], "cos_apical_negg": [0.047639235854148865, 0.054375506937503815, 0.11665451526641846], "cos_Ac_negg": [0.5135130882263184, 0.493655800819397, 0.4611470699310303], "r_norm": [0.42564013600349426, 0.4166065454483032, 0.41495829820632935]}, {"step": 700, "epoch": 1, "train_loss": 0.19576694071292877, "cos_r_negg": [0.5116797089576721, 0.5149182081222534, 0.4854634404182434], "cos_innovation_negg": [0.5116797089576721, 0.5149182081222534, 0.4854634404182434], "cos_apical_negg": [0.04140525683760643, 0.06269320100545883, 0.11289717257022858], "cos_Ac_negg": [0.5239114761352539, 0.5192609429359436, 0.49155494570732117], "r_norm": [0.4615369439125061, 0.4475287199020386, 0.4478917717933655]}, {"step": 800, "epoch": 1, "train_loss": 0.13500498235225677, "cos_r_negg": [0.5412785410881042, 0.5338720083236694, 0.49602705240249634], "cos_innovation_negg": [0.5412785410881042, 0.5338720083236694, 0.49602705240249634], "cos_apical_negg": [0.042619746178388596, 0.062221676111221313, 0.09834045171737671], "cos_Ac_negg": [0.5519948601722717, 0.5353431105613708, 0.49804800748825073], "r_norm": [0.36324626207351685, 0.3547770380973816, 0.35415929555892944]}, {"step": 900, "epoch": 1, "train_loss": 0.19931073486804962, "cos_r_negg": [0.5573374629020691, 0.5556252002716064, 0.49063265323638916], "cos_innovation_negg": [0.5573374629020691, 0.5556252002716064, 0.49063265323638916], "cos_apical_negg": [0.04584997147321701, 0.06994988024234772, 0.10097363591194153], "cos_Ac_negg": [0.560693621635437, 0.5552884936332703, 0.488823801279068], "r_norm": [0.43848469853401184, 0.4289107024669647, 0.4265971779823303]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9536, "test_loss": 0.1556456199645996}, {"step": 1000, "epoch": 2, "train_loss": 0.04337725415825844, "cos_r_negg": [0.5406967401504517, 0.5441626906394958, 0.5123414993286133], "cos_innovation_negg": [0.5406967401504517, 0.5441626906394958, 0.5123414993286133], "cos_apical_negg": [0.02406124398112297, 0.052928850054740906, 0.10016977787017822], "cos_Ac_negg": [0.5500516891479492, 0.5479145050048828, 0.513395369052887], "r_norm": [0.3558533787727356, 0.34642890095710754, 0.3454371988773346], "ldr": {"dL_sdil": -0.010897994041442871, "dL_bp": -0.00895872712135315, "ratio": 1.2164667919695282}}, {"step": 1100, "epoch": 2, "train_loss": 0.12945353984832764, "cos_r_negg": [0.540961503982544, 0.547196626663208, 0.5078578591346741], "cos_innovation_negg": [0.540961503982544, 0.547196626663208, 0.5078578591346741], "cos_apical_negg": [0.035796165466308594, 0.048418864607810974, 0.1101682037115097], "cos_Ac_negg": [0.5532693862915039, 0.5568880438804626, 0.5110524892807007], "r_norm": [0.332541286945343, 0.3214121162891388, 0.32319897413253784]}, {"step": 1200, "epoch": 2, "train_loss": 0.12910637259483337, "cos_r_negg": [0.5410082936286926, 0.5529738664627075, 0.5163585543632507], "cos_innovation_negg": [0.5410082936286926, 0.5529738664627075, 0.5163585543632507], "cos_apical_negg": [0.01959778368473053, 0.039587631821632385, 0.11256879568099976], "cos_Ac_negg": [0.5523759126663208, 0.5613055229187012, 0.5173860788345337], "r_norm": [0.3318207263946533, 0.3226848244667053, 0.3240436911582947]}, {"step": 1300, "epoch": 2, "train_loss": 0.10018610954284668, "cos_r_negg": [0.5347980856895447, 0.5633956789970398, 0.5126184225082397], "cos_innovation_negg": [0.5347980856895447, 0.5633956789970398, 0.5126184225082397], "cos_apical_negg": [0.02419588342308998, 0.056053582578897476, 0.10105574876070023], "cos_Ac_negg": [0.5439138412475586, 0.5699706077575684, 0.5123409032821655], "r_norm": [0.32683128118515015, 0.31890031695365906, 0.31658512353897095]}, {"step": 1400, "epoch": 2, "train_loss": 0.1301070749759674, "cos_r_negg": [0.5010162591934204, 0.5402288436889648, 0.5228582620620728], "cos_innovation_negg": [0.5010162591934204, 0.5402288436889648, 0.5228582620620728], "cos_apical_negg": [0.00686885230243206, 0.04561745002865791, 0.10342801362276077], "cos_Ac_negg": [0.5327618718147278, 0.5623195171356201, 0.5346428155899048], "r_norm": [0.2709745168685913, 0.2633535861968994, 0.26226699352264404]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9658, "test_loss": 0.11943423690795898}, {"step": 1500, "epoch": 3, "train_loss": 0.09488650411367416, "cos_r_negg": [0.5477433204650879, 0.5659247040748596, 0.5450236797332764], "cos_innovation_negg": [0.5477433204650879, 0.5659247040748596, 0.5450236797332764], "cos_apical_negg": [0.02507450059056282, 0.04850952327251434, 0.10936501622200012], "cos_Ac_negg": [0.5674236416816711, 0.58188796043396, 0.5530956983566284], "r_norm": [0.25747135281562805, 0.2512120306491852, 0.25033754110336304], "ldr": {"dL_sdil": -0.005412191152572632, "dL_bp": -0.005684912204742432, "ratio": 0.9520272183021063}}, {"step": 1600, "epoch": 3, "train_loss": 0.09669487923383713, "cos_r_negg": [0.5311207175254822, 0.5536269545555115, 0.5428143739700317], "cos_innovation_negg": [0.5311207175254822, 0.5536269545555115, 0.5428143739700317], "cos_apical_negg": [0.028560880571603775, 0.04665552079677582, 0.10565381497144699], "cos_Ac_negg": [0.5533983111381531, 0.5779848098754883, 0.5536564588546753], "r_norm": [0.290454238653183, 0.2804765999317169, 0.282884418964386]}, {"step": 1700, "epoch": 3, "train_loss": 0.046024542301893234, "cos_r_negg": [0.5218465328216553, 0.5565448999404907, 0.5266669988632202], "cos_innovation_negg": [0.5218465328216553, 0.5565448999404907, 0.5266669988632202], "cos_apical_negg": [0.013521810993552208, 0.05554120987653732, 0.10070943832397461], "cos_Ac_negg": [0.5539103746414185, 0.5861827731132507, 0.5446239709854126], "r_norm": [0.2801855206489563, 0.27018672227859497, 0.2713976502418518]}, {"step": 1800, "epoch": 3, "train_loss": 0.011728792451322079, "cos_r_negg": [0.52141273021698, 0.540260910987854, 0.5321792960166931], "cos_innovation_negg": [0.52141273021698, 0.540260910987854, 0.5321792960166931], "cos_apical_negg": [0.020364152267575264, 0.054958440363407135, 0.11829151213169098], "cos_Ac_negg": [0.5664126873016357, 0.5849656462669373, 0.5539586544036865], "r_norm": [0.253146767616272, 0.24607858061790466, 0.24518734216690063]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.967, "test_loss": 0.11927742767333985}, {"step": 1900, "epoch": 4, "train_loss": 0.030803844332695007, "cos_r_negg": [0.5550085306167603, 0.5748291015625, 0.5515085458755493], "cos_innovation_negg": [0.5550085306167603, 0.5748291015625, 0.5515085458755493], "cos_apical_negg": [0.011234588921070099, 0.04739664867520332, 0.1076519712805748], "cos_Ac_negg": [0.5830277800559998, 0.5995380878448486, 0.563378095626831], "r_norm": [0.2503376603126526, 0.24449379742145538, 0.24348139762878418]}, {"step": 2000, "epoch": 4, "train_loss": 0.09435570985078812, "cos_r_negg": [0.5165828466415405, 0.5491183996200562, 0.5441352128982544], "cos_innovation_negg": [0.5165828466415405, 0.5491183996200562, 0.5441352128982544], "cos_apical_negg": [-0.005702102556824684, 0.04362977296113968, 0.10505573451519012], "cos_Ac_negg": [0.5735284090042114, 0.5990222692489624, 0.5746065974235535], "r_norm": [0.20219786465168, 0.19464869797229767, 0.19419066607952118], "ldr": {"dL_sdil": -0.004922345280647278, "dL_bp": -0.004920199513435364, "ratio": 1.0004361138620608}}, {"step": 2100, "epoch": 4, "train_loss": 0.008065913803875446, "cos_r_negg": [0.5316856503486633, 0.5577585697174072, 0.5528817772865295], "cos_innovation_negg": [0.5316856503486633, 0.5577585697174072, 0.5528817772865295], "cos_apical_negg": [0.018969878554344177, 0.044544756412506104, 0.11087940633296967], "cos_Ac_negg": [0.5732593536376953, 0.5963599681854248, 0.5710786581039429], "r_norm": [0.24292318522930145, 0.23432767391204834, 0.23457136750221252]}, {"step": 2200, "epoch": 4, "train_loss": 0.0065501658245921135, "cos_r_negg": [0.5224059820175171, 0.5490351319313049, 0.5424318313598633], "cos_innovation_negg": [0.5224059820175171, 0.5490351319313049, 0.5424318313598633], "cos_apical_negg": [0.014763617888092995, 0.03863360360264778, 0.09883178770542145], "cos_Ac_negg": [0.5712668299674988, 0.6000847220420837, 0.5719034671783447], "r_norm": [0.219838947057724, 0.21344280242919922, 0.21355140209197998]}, {"step": 2300, "epoch": 4, "train_loss": 0.14067436754703522, "cos_r_negg": [0.49908602237701416, 0.5320106744766235, 0.5351196527481079], "cos_innovation_negg": [0.49908602237701416, 0.5320106744766235, 0.5351196527481079], "cos_apical_negg": [0.008820908144116402, 0.048831529915332794, 0.10290086269378662], "cos_Ac_negg": [0.5642155408859253, 0.5992565155029297, 0.5796096324920654], "r_norm": [0.2458912581205368, 0.23886513710021973, 0.23990841209888458]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9661, "test_loss": 0.1243594123840332}, {"step": 2400, "epoch": 5, "train_loss": 0.0868406593799591, "cos_r_negg": [0.4981728494167328, 0.5238622426986694, 0.5275735259056091], "cos_innovation_negg": [0.4981728494167328, 0.5238622426986694, 0.5275735259056091], "cos_apical_negg": [0.008215553127229214, 0.04122934117913246, 0.10164561867713928], "cos_Ac_negg": [0.5746393203735352, 0.6053228378295898, 0.5804059505462646], "r_norm": [0.20620393753051758, 0.19982397556304932, 0.19935940206050873]}, {"step": 2500, "epoch": 5, "train_loss": 0.040987178683280945, "cos_r_negg": [0.4637001156806946, 0.486184298992157, 0.49550023674964905], "cos_innovation_negg": [0.4637001156806946, 0.486184298992157, 0.49550023674964905], "cos_apical_negg": [0.01503700576722622, 0.045721884816884995, 0.09604103118181229], "cos_Ac_negg": [0.5619102120399475, 0.6024391651153564, 0.579123854637146], "r_norm": [0.19776150584220886, 0.1923750638961792, 0.19342955946922302], "ldr": {"dL_sdil": -0.005911648273468018, "dL_bp": -0.00665283203125, "ratio": 0.8885912413990825}}, {"step": 2600, "epoch": 5, "train_loss": 0.03735675662755966, "cos_r_negg": [0.48341357707977295, 0.5146516561508179, 0.5274155139923096], "cos_innovation_negg": [0.48341357707977295, 0.5146516561508179, 0.5274155139923096], "cos_apical_negg": [0.00393025204539299, 0.04374466463923454, 0.10936721414327621], "cos_Ac_negg": [0.590373694896698, 0.6134263277053833, 0.5952221155166626], "r_norm": [0.2101920247077942, 0.20476499199867249, 0.20305541157722473]}, {"step": 2700, "epoch": 5, "train_loss": 0.03785250335931778, "cos_r_negg": [0.4948693811893463, 0.5172471404075623, 0.5347311496734619], "cos_innovation_negg": [0.4948693811893463, 0.5172471404075623, 0.5347311496734619], "cos_apical_negg": [0.004449144005775452, 0.04389144480228424, 0.11013566702604294], "cos_Ac_negg": [0.602059006690979, 0.6207597255706787, 0.6133108735084534], "r_norm": [0.21967127919197083, 0.21421346068382263, 0.2138521820306778]}, {"step": 2800, "epoch": 5, "train_loss": 0.07534697651863098, "cos_r_negg": [0.4680801033973694, 0.501079797744751, 0.5174782276153564], "cos_innovation_negg": [0.4680801033973694, 0.501079797744751, 0.5174782276153564], "cos_apical_negg": [0.008842382580041885, 0.03690087050199509, 0.10078456252813339], "cos_Ac_negg": [0.5848636627197266, 0.6153844594955444, 0.6031122207641602], "r_norm": [0.2104223370552063, 0.2037733793258667, 0.20278412103652954]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9694, "test_loss": 0.11499397735595702}, {"step": 2900, "epoch": 6, "train_loss": 0.0019426055951043963, "cos_r_negg": [0.4577125906944275, 0.48318934440612793, 0.5011618733406067], "cos_innovation_negg": [0.4577125906944275, 0.48318934440612793, 0.5011618733406067], "cos_apical_negg": [0.020965497940778732, 0.04053390026092529, 0.10396619886159897], "cos_Ac_negg": [0.5896856188774109, 0.6151741147041321, 0.604279100894928], "r_norm": [0.20870232582092285, 0.20246008038520813, 0.20229654014110565]}, {"step": 3000, "epoch": 6, "train_loss": 0.06829206645488739, "cos_r_negg": [0.446952223777771, 0.4674347937107086, 0.4866243898868561], "cos_innovation_negg": [0.446952223777771, 0.4674347937107086, 0.4866243898868561], "cos_apical_negg": [-0.009396068751811981, 0.03873144090175629, 0.09478671848773956], "cos_Ac_negg": [0.6048896312713623, 0.6227243542671204, 0.6129453182220459], "r_norm": [0.1770733892917633, 0.17196708917617798, 0.17103160917758942], "ldr": {"dL_sdil": -0.004875198006629944, "dL_bp": -0.005990125238895416, "ratio": 0.8138724671354174}}, {"step": 3100, "epoch": 6, "train_loss": 0.012289129197597504, "cos_r_negg": [0.4534164071083069, 0.4699392318725586, 0.497898131608963], "cos_innovation_negg": [0.4534164071083069, 0.4699392318725586, 0.497898131608963], "cos_apical_negg": [0.01680128276348114, 0.034895673394203186, 0.10088913887739182], "cos_Ac_negg": [0.6157838106155396, 0.6316499710083008, 0.6197147369384766], "r_norm": [0.17025022208690643, 0.16576413810253143, 0.16465052962303162]}, {"step": 3200, "epoch": 6, "train_loss": 0.030991695821285248, "cos_r_negg": [0.4131392240524292, 0.44730043411254883, 0.4769807457923889], "cos_innovation_negg": [0.4131392240524292, 0.44730043411254883, 0.4769807457923889], "cos_apical_negg": [0.00890885479748249, 0.03517932444810867, 0.09522219002246857], "cos_Ac_negg": [0.5946764945983887, 0.6279523372650146, 0.6239750981330872], "r_norm": [0.16218790411949158, 0.15704484283924103, 0.15644490718841553]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9685, "test_loss": 0.130343825340271}, {"step": 3300, "epoch": 7, "train_loss": 0.004070380236953497, "cos_r_negg": [0.40077900886535645, 0.4304547607898712, 0.4595605731010437], "cos_innovation_negg": [0.40077900886535645, 0.4304547607898712, 0.4595605731010437], "cos_apical_negg": [0.005518389865756035, 0.034604042768478394, 0.09811795502901077], "cos_Ac_negg": [0.5955986976623535, 0.6299235820770264, 0.6221448183059692], "r_norm": [0.14800630509853363, 0.14408552646636963, 0.14225590229034424]}, {"step": 3400, "epoch": 7, "train_loss": 0.02358859032392502, "cos_r_negg": [0.3879971504211426, 0.411190927028656, 0.4450938105583191], "cos_innovation_negg": [0.3879971504211426, 0.411190927028656, 0.4450938105583191], "cos_apical_negg": [0.016526248306035995, 0.035195671021938324, 0.10495409369468689], "cos_Ac_negg": [0.6020306348800659, 0.6280429363250732, 0.6263999342918396], "r_norm": [0.17126505076885223, 0.1664196103811264, 0.1652672290802002]}, {"step": 3500, "epoch": 7, "train_loss": 0.001445534871891141, "cos_r_negg": [0.3632965087890625, 0.402198851108551, 0.4473251700401306], "cos_innovation_negg": [0.3632965087890625, 0.402198851108551, 0.4473251700401306], "cos_apical_negg": [0.009577827528119087, 0.028438780456781387, 0.10632304847240448], "cos_Ac_negg": [0.5836133360862732, 0.6256428956985474, 0.635414719581604], "r_norm": [0.16099384427070618, 0.15612384676933289, 0.15620404481887817], "ldr": {"dL_sdil": -0.005360133945941925, "dL_bp": -0.007305599749088287, "ratio": 0.7337021093457591}}, {"step": 3600, "epoch": 7, "train_loss": 0.0047991215251386166, "cos_r_negg": [0.37746262550354004, 0.40822017192840576, 0.45053255558013916], "cos_innovation_negg": [0.37746262550354004, 0.40822017192840576, 0.45053255558013916], "cos_apical_negg": [0.00926753506064415, 0.039603784680366516, 0.10520198196172714], "cos_Ac_negg": [0.5973981022834778, 0.6330288648605347, 0.6374830007553101], "r_norm": [0.17061659693717957, 0.1651216298341751, 0.1666044294834137]}, {"step": 3700, "epoch": 7, "train_loss": 0.0014059263048693538, "cos_r_negg": [0.36543941497802734, 0.39931243658065796, 0.4484252333641052], "cos_innovation_negg": [0.36543941497802734, 0.39931243658065796, 0.4484252333641052], "cos_apical_negg": [0.010484463535249233, 0.03964608535170555, 0.10806763917207718], "cos_Ac_negg": [0.5976572632789612, 0.6345747709274292, 0.6466951370239258], "r_norm": [0.1540808379650116, 0.14826953411102295, 0.14882683753967285]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9716, "test_loss": 0.11724653453826904}, {"step": 3800, "epoch": 8, "train_loss": 0.0008633130346424878, "cos_r_negg": [0.3445333242416382, 0.377445250749588, 0.4279426336288452], "cos_innovation_negg": [0.3445333242416382, 0.377445250749588, 0.4279426336288452], "cos_apical_negg": [0.0023265022318810225, 0.0419115386903286, 0.10405996441841125], "cos_Ac_negg": [0.5913555026054382, 0.6292985677719116, 0.6440975666046143], "r_norm": [0.15268784761428833, 0.14833927154541016, 0.14836238324642181]}, {"step": 3900, "epoch": 8, "train_loss": 0.016011923551559448, "cos_r_negg": [0.30991411209106445, 0.3465301990509033, 0.4040083587169647], "cos_innovation_negg": [0.30991411209106445, 0.3465301990509033, 0.4040083587169647], "cos_apical_negg": [-0.000366229098290205, 0.03983292356133461, 0.10650406777858734], "cos_Ac_negg": [0.5823796987533569, 0.6275235414505005, 0.6463069915771484], "r_norm": [0.1643437147140503, 0.1599186658859253, 0.15844371914863586]}, {"step": 4000, "epoch": 8, "train_loss": 0.005876308307051659, "cos_r_negg": [0.2810835540294647, 0.32485589385032654, 0.38159894943237305], "cos_innovation_negg": [0.2810835540294647, 0.32485589385032654, 0.38159894943237305], "cos_apical_negg": [-0.004385711625218391, 0.03626742213964462, 0.10575808584690094], "cos_Ac_negg": [0.5818820595741272, 0.6265275478363037, 0.6461528539657593], "r_norm": [0.15862736105918884, 0.15402765572071075, 0.15324923396110535], "ldr": {"dL_sdil": -0.0052199587225914, "dL_bp": -0.00768677145242691, "ratio": 0.679083377839122}}, {"step": 4100, "epoch": 8, "train_loss": 0.009995711036026478, "cos_r_negg": [0.2915729284286499, 0.32985246181488037, 0.3921487331390381], "cos_innovation_negg": [0.2915729284286499, 0.32985246181488037, 0.3921487331390381], "cos_apical_negg": [-0.004570425022393465, 0.03635071963071823, 0.10753794014453888], "cos_Ac_negg": [0.5940312147140503, 0.6342158913612366, 0.6535048484802246], "r_norm": [0.174652099609375, 0.16777926683425903, 0.16745731234550476]}, {"step": 4200, "epoch": 8, "train_loss": 0.01584916189312935, "cos_r_negg": [0.2785198986530304, 0.3130256235599518, 0.37501060962677], "cos_innovation_negg": [0.2785198986530304, 0.3130256235599518, 0.37501060962677], "cos_apical_negg": [0.0033586015924811363, 0.034764379262924194, 0.10399192571640015], "cos_Ac_negg": [0.5832823514938354, 0.627549409866333, 0.6530169248580933], "r_norm": [0.14530107378959656, 0.14131084084510803, 0.14038027822971344]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.972, "test_loss": 0.13099193515777588}, {"step": 4300, "epoch": 9, "train_loss": 0.0013420312898233533, "cos_r_negg": [0.27485042810440063, 0.3154665231704712, 0.37518662214279175], "cos_innovation_negg": [0.27485042810440063, 0.3154665231704712, 0.37518662214279175], "cos_apical_negg": [-0.0016189543530344963, 0.038209930062294006, 0.10528067499399185], "cos_Ac_negg": [0.5919089317321777, 0.6336437463760376, 0.657653272151947], "r_norm": [0.15844403207302094, 0.15488797426223755, 0.15405455231666565]}, {"step": 4400, "epoch": 9, "train_loss": 0.0013598615769296885, "cos_r_negg": [0.27434292435646057, 0.3129573464393616, 0.3681192398071289], "cos_innovation_negg": [0.27434292435646057, 0.3129573464393616, 0.3681192398071289], "cos_apical_negg": [-0.002546069212257862, 0.037157535552978516, 0.10349668562412262], "cos_Ac_negg": [0.5906094908714294, 0.6344246864318848, 0.6580358743667603], "r_norm": [0.1486673355102539, 0.14501874148845673, 0.1442796289920807]}, {"step": 4500, "epoch": 9, "train_loss": 0.03456545248627663, "cos_r_negg": [0.25133368372917175, 0.28834646940231323, 0.3497832417488098], "cos_innovation_negg": [0.25133368372917175, 0.28834646940231323, 0.3497832417488098], "cos_apical_negg": [0.0015513217076659203, 0.03367505967617035, 0.10306073725223541], "cos_Ac_negg": [0.5931183099746704, 0.6332410573959351, 0.6618243455886841], "r_norm": [0.15447081625461578, 0.1503368765115738, 0.14995679259300232], "ldr": {"dL_sdil": -0.0061156973242759705, "dL_bp": -0.009683042764663696, "ratio": 0.6315883832088369}}, {"step": 4600, "epoch": 9, "train_loss": 0.000631060975138098, "cos_r_negg": [0.25374943017959595, 0.2884773910045624, 0.35207968950271606], "cos_innovation_negg": [0.25374943017959595, 0.2884773910045624, 0.35207968950271606], "cos_apical_negg": [0.0015299245715141296, 0.031631648540496826, 0.10145542025566101], "cos_Ac_negg": [0.5881377458572388, 0.6348575353622437, 0.6593854427337646], "r_norm": [0.15295439958572388, 0.14834043383598328, 0.14849190413951874]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9716, "test_loss": 0.13195653724670411}, {"step": 4700, "epoch": 10, "train_loss": 0.0018224127124994993, "cos_r_negg": [0.24005019664764404, 0.27142199873924255, 0.33488965034484863], "cos_innovation_negg": [0.24005019664764404, 0.27142199873924255, 0.33488965034484863], "cos_apical_negg": [0.004340329673141241, 0.03361248970031738, 0.09669365733861923], "cos_Ac_negg": [0.592002272605896, 0.637415885925293, 0.6620798110961914], "r_norm": [0.1530528962612152, 0.14844045042991638, 0.14824894070625305]}, {"step": 4800, "epoch": 10, "train_loss": 0.00042262973147444427, "cos_r_negg": [0.23037979006767273, 0.26626986265182495, 0.3306382894515991], "cos_innovation_negg": [0.23037979006767273, 0.26626986265182495, 0.3306382894515991], "cos_apical_negg": [-0.00018684379756450653, 0.03444229066371918, 0.09747174382209778], "cos_Ac_negg": [0.5839577913284302, 0.6326618790626526, 0.6625460386276245], "r_norm": [0.15761873126029968, 0.1525515615940094, 0.15248043835163116]}, {"step": 4900, "epoch": 10, "train_loss": 0.002176593756303191, "cos_r_negg": [0.22571918368339539, 0.26211169362068176, 0.32274991273880005], "cos_innovation_negg": [0.22571918368339539, 0.26211169362068176, 0.32274991273880005], "cos_apical_negg": [-0.002524089068174362, 0.036017339676618576, 0.09383861720561981], "cos_Ac_negg": [0.5886404514312744, 0.6347569227218628, 0.6628813743591309], "r_norm": [0.1424628496170044, 0.1377774327993393, 0.13758273422718048]}, {"step": 5000, "epoch": 10, "train_loss": 0.0006744620623067021, "cos_r_negg": [0.2222062051296234, 0.25270354747772217, 0.30892008543014526], "cos_innovation_negg": [0.2222062051296234, 0.25270354747772217, 0.30892008543014526], "cos_apical_negg": [0.0009590340778231621, 0.03153621777892113, 0.08860278129577637], "cos_Ac_negg": [0.5725927352905273, 0.6265392303466797, 0.6623731851577759], "r_norm": [0.15401682257652283, 0.15014326572418213, 0.1496882438659668], "ldr": {"dL_sdil": -0.006028294563293457, "dL_bp": -0.009351417422294617, "ratio": 0.6446396616754015}}, {"step": 5100, "epoch": 10, "train_loss": 0.0014848911669105291, "cos_r_negg": [0.21961820125579834, 0.2580984830856323, 0.30639171600341797], "cos_innovation_negg": [0.21961820125579834, 0.2580984830856323, 0.30639171600341797], "cos_apical_negg": [-0.0019001984037458897, 0.03560680150985718, 0.08627481758594513], "cos_Ac_negg": [0.5766924619674683, 0.6369302868843079, 0.6644871234893799], "r_norm": [0.1511916220188141, 0.14784136414527893, 0.14697614312171936]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9721, "test_loss": 0.1368281675338745}, {"step": 5200, "epoch": 11, "train_loss": 0.0008225785568356514, "cos_r_negg": [0.21912850439548492, 0.25657331943511963, 0.30576181411743164], "cos_innovation_negg": [0.21912850439548492, 0.25657331943511963, 0.30576181411743164], "cos_apical_negg": [-0.0035089156590402126, 0.0334937609732151, 0.08607956022024155], "cos_Ac_negg": [0.5823262929916382, 0.6361823081970215, 0.664984941482544], "r_norm": [0.13863006234169006, 0.1350957751274109, 0.134585902094841]}, {"step": 5300, "epoch": 11, "train_loss": 0.002895822050049901, "cos_r_negg": [0.21578525006771088, 0.24931523203849792, 0.29838377237319946], "cos_innovation_negg": [0.21578525006771088, 0.24931523203849792, 0.29838377237319946], "cos_apical_negg": [-0.0028217416256666183, 0.03173669055104256, 0.08159370720386505], "cos_Ac_negg": [0.5741730332374573, 0.629033625125885, 0.6636229157447815], "r_norm": [0.1457689106464386, 0.1420048177242279, 0.14134639501571655]}, {"step": 5400, "epoch": 11, "train_loss": 0.0014625702751800418, "cos_r_negg": [0.21218493580818176, 0.24303241074085236, 0.29275012016296387], "cos_innovation_negg": [0.21218493580818176, 0.24303241074085236, 0.29275012016296387], "cos_apical_negg": [-0.0010529994033277035, 0.031591467559337616, 0.08191727101802826], "cos_Ac_negg": [0.5728154182434082, 0.6315698623657227, 0.6656259894371033], "r_norm": [0.14087790250778198, 0.1373642235994339, 0.13662117719650269]}, {"step": 5500, "epoch": 11, "train_loss": 0.0016903358045965433, "cos_r_negg": [0.21171659231185913, 0.24361832439899445, 0.29311251640319824], "cos_innovation_negg": [0.21171659231185913, 0.24361832439899445, 0.29311251640319824], "cos_apical_negg": [-0.001549859531223774, 0.03345705568790436, 0.08360060304403305], "cos_Ac_negg": [0.5768427848815918, 0.6334128975868225, 0.6679232120513916], "r_norm": [0.13572141528129578, 0.1321655958890915, 0.13195744156837463], "ldr": {"dL_sdil": -0.005469560623168945, "dL_bp": -0.009053520858287811, "ratio": 0.6041363032992825}}, {"step": 5600, "epoch": 11, "train_loss": 0.0012983162887394428, "cos_r_negg": [0.20741267502307892, 0.23724700510501862, 0.28603723645210266], "cos_innovation_negg": [0.20741267502307892, 0.23724700510501862, 0.28603723645210266], "cos_apical_negg": [-0.002379343379288912, 0.03247354179620743, 0.08056950569152832], "cos_Ac_negg": [0.5771089196205139, 0.6324926614761353, 0.6671242713928223], "r_norm": [0.13510887324810028, 0.13125915825366974, 0.13129353523254395]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9727, "test_loss": 0.14068544807434083}, {"step": 5700, "epoch": 12, "train_loss": 0.0007156288484111428, "cos_r_negg": [0.19856968522071838, 0.23168379068374634, 0.2786247134208679], "cos_innovation_negg": [0.19856968522071838, 0.23168379068374634, 0.2786247134208679], "cos_apical_negg": [-0.0016147242859005928, 0.03143199533224106, 0.07900910079479218], "cos_Ac_negg": [0.5706033706665039, 0.6268273591995239, 0.6659523844718933], "r_norm": [0.1336199790239334, 0.12988582253456116, 0.12971554696559906]}, {"step": 5800, "epoch": 12, "train_loss": 0.00048588408390060067, "cos_r_negg": [0.1939568668603897, 0.23039600253105164, 0.27736300230026245], "cos_innovation_negg": [0.1939568668603897, 0.23039600253105164, 0.27736300230026245], "cos_apical_negg": [-0.0022492785938084126, 0.03173355013132095, 0.08023496717214584], "cos_Ac_negg": [0.5727983713150024, 0.6307904720306396, 0.6693071126937866], "r_norm": [0.1406402438879013, 0.13692748546600342, 0.13653405010700226]}, {"step": 5900, "epoch": 12, "train_loss": 0.0019222944974899292, "cos_r_negg": [0.19307759404182434, 0.23198121786117554, 0.2767632007598877], "cos_innovation_negg": [0.19307759404182434, 0.23198121786117554, 0.2767632007598877], "cos_apical_negg": [-0.004706338047981262, 0.03159979730844498, 0.07717008888721466], "cos_Ac_negg": [0.5650250911712646, 0.6260983943939209, 0.6689882278442383], "r_norm": [0.13981205224990845, 0.1357787847518921, 0.13550418615341187]}, {"step": 6000, "epoch": 12, "train_loss": 0.00045124097960069776, "cos_r_negg": [0.1851966679096222, 0.2210320681333542, 0.2665230631828308], "cos_innovation_negg": [0.1851966679096222, 0.2210320681333542, 0.2665230631828308], "cos_apical_negg": [-0.0034684371203184128, 0.031309306621551514, 0.07571516931056976], "cos_Ac_negg": [0.5698431730270386, 0.6257672309875488, 0.6678767204284668], "r_norm": [0.13368269801139832, 0.12989981472492218, 0.12956127524375916], "ldr": {"dL_sdil": -0.005572102963924408, "dL_bp": -0.00944441556930542, "ratio": 0.5899891764646483}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9721, "test_loss": 0.143893613243103}, {"step": 6100, "epoch": 13, "train_loss": 0.0005720598273910582, "cos_r_negg": [0.18247531354427338, 0.21841880679130554, 0.26117002964019775], "cos_innovation_negg": [0.18247531354427338, 0.21841880679130554, 0.26117002964019775], "cos_apical_negg": [-0.008093561977148056, 0.03186175972223282, 0.07311908900737762], "cos_Ac_negg": [0.5712630152702332, 0.6270120143890381, 0.6698102951049805], "r_norm": [0.13618148863315582, 0.13248328864574432, 0.13214844465255737]}, {"step": 6200, "epoch": 13, "train_loss": 0.0006833100924268365, "cos_r_negg": [0.1804307997226715, 0.21572285890579224, 0.25849825143814087], "cos_innovation_negg": [0.1804307997226715, 0.21572285890579224, 0.25849825143814087], "cos_apical_negg": [-0.004912731237709522, 0.031982652842998505, 0.07271470874547958], "cos_Ac_negg": [0.5690498352050781, 0.6263390779495239, 0.6701257824897766], "r_norm": [0.1367523968219757, 0.1327851414680481, 0.1326279640197754]}, {"step": 6300, "epoch": 13, "train_loss": 0.0008542236755602062, "cos_r_negg": [0.17706552147865295, 0.2127162516117096, 0.25362932682037354], "cos_innovation_negg": [0.17706552147865295, 0.2127162516117096, 0.25362932682037354], "cos_apical_negg": [-0.006028501316905022, 0.03235006332397461, 0.07116851955652237], "cos_Ac_negg": [0.566336989402771, 0.6259113550186157, 0.670574963092804], "r_norm": [0.1369779109954834, 0.1333196461200714, 0.13319005072116852]}, {"step": 6400, "epoch": 13, "train_loss": 0.0005410106969065964, "cos_r_negg": [0.18153151869773865, 0.21716755628585815, 0.25861719250679016], "cos_innovation_negg": [0.18153151869773865, 0.21716755628585815, 0.25861719250679016], "cos_apical_negg": [-0.005879516247659922, 0.031762123107910156, 0.07145000249147415], "cos_Ac_negg": [0.5664576292037964, 0.6267338395118713, 0.6720196008682251], "r_norm": [0.13243013620376587, 0.12902915477752686, 0.12883922457695007]}, {"step": 6500, "epoch": 13, "train_loss": 0.00020985145238228142, "cos_r_negg": [0.17772656679153442, 0.21242636442184448, 0.25414592027664185], "cos_innovation_negg": [0.17772656679153442, 0.21242636442184448, 0.25414592027664185], "cos_apical_negg": [-0.0075380546040833, 0.030885789543390274, 0.06898142397403717], "cos_Ac_negg": [0.5650582313537598, 0.6260440349578857, 0.6713216304779053], "r_norm": [0.1292339712381363, 0.12573862075805664, 0.12550702691078186], "ldr": {"dL_sdil": -0.0056751444935798645, "dL_bp": -0.009983845055103302, "ratio": 0.5684327493322806}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9727, "test_loss": 0.1467740493774414}, {"step": 6600, "epoch": 14, "train_loss": 0.00017112368368543684, "cos_r_negg": [0.17454290390014648, 0.2121538668870926, 0.2518329918384552], "cos_innovation_negg": [0.17454290390014648, 0.2121538668870926, 0.2518329918384552], "cos_apical_negg": [-0.007086961530148983, 0.03190644085407257, 0.0692686215043068], "cos_Ac_negg": [0.5636143684387207, 0.6269121766090393, 0.6718631982803345], "r_norm": [0.12923821806907654, 0.1260632574558258, 0.125556081533432]}, {"step": 6700, "epoch": 14, "train_loss": 0.0006778803071938455, "cos_r_negg": [0.17567873001098633, 0.2124112993478775, 0.2522124648094177], "cos_innovation_negg": [0.17567873001098633, 0.2124112993478775, 0.2522124648094177], "cos_apical_negg": [-0.006559518165886402, 0.030668459832668304, 0.06823772192001343], "cos_Ac_negg": [0.5630702972412109, 0.6271776556968689, 0.6719936728477478], "r_norm": [0.13151130080223083, 0.12808698415756226, 0.12786173820495605]}, {"step": 6800, "epoch": 14, "train_loss": 0.0005970008205622435, "cos_r_negg": [0.1692158281803131, 0.20591729879379272, 0.24470087885856628], "cos_innovation_negg": [0.1692158281803131, 0.20591729879379272, 0.24470087885856628], "cos_apical_negg": [-0.00712909922003746, 0.03100281022489071, 0.06647836416959763], "cos_Ac_negg": [0.5601401329040527, 0.6256040334701538, 0.6721820831298828], "r_norm": [0.12940508127212524, 0.1262277215719223, 0.12584228813648224]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005270083784125745, "cos_r_negg": [0.16722643375396729, 0.20353162288665771, 0.24128787219524384], "cos_innovation_negg": [0.16722643375396729, 0.20353162288665771, 0.24128787219524384], "cos_apical_negg": [-0.0067224702797830105, 0.0305806752294302, 0.06410964578390121], "cos_Ac_negg": [0.5593805313110352, 0.625951886177063, 0.6732284426689148], "r_norm": [0.1306028664112091, 0.12735921144485474, 0.12685713171958923]}, {"step": 7000, "epoch": 14, "train_loss": 0.0004866862145718187, "cos_r_negg": [0.1670301854610443, 0.2021043747663498, 0.24019977450370789], "cos_innovation_negg": [0.1670301854610443, 0.2021043747663498, 0.24019977450370789], "cos_apical_negg": [-0.0063683222979307175, 0.03093770518898964, 0.06457505375146866], "cos_Ac_negg": [0.5627425909042358, 0.6281017065048218, 0.6729280352592468], "r_norm": [0.1315777450799942, 0.12827616930007935, 0.12790822982788086], "ldr": {"dL_sdil": -0.005945973098278046, "dL_bp": -0.010425247251987457, "ratio": 0.5703436047662575}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9721, "test_loss": 0.15021902580261232}], "final": {"test_acc": 0.9721, "test_loss": 0.15021902580261232, "wall_s": 23.304119110107422, "cos_r_negg": [0.16458654403686523, 0.2002539038658142, 0.2374999225139618], "cos_innovation_negg": [0.16458654403686523, 0.2002539038658142, 0.2374999225139618], "cos_apical_negg": [-0.008399410173296928, 0.030416259542107582, 0.06369166821241379], "cos_Ac_negg": [0.5622594356536865, 0.6273370981216431, 0.6730587482452393]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_residual_s1.json b/results/nuis_ctrl_v1_rho0p5_residual_s1.json new file mode 100644 index 0000000..2934eac --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_residual_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_residual_s1", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.327335834503174, "cos_r_negg": [0.011923439800739288, -0.013784728944301605, -0.005784478038549423], "cos_innovation_negg": [0.011923439800739288, -0.013784728944301605, -0.005784478038549423], "cos_apical_negg": [0.0065222978591918945, -0.010908648371696472, -0.008246229030191898], "cos_Ac_negg": [0.01192301046103239, -0.013784660957753658, -0.0057846191339194775], "r_norm": [4.739156723022461, 4.7983598709106445, 4.788495063781738], "ldr": {"dL_sdil": -0.04239368438720703, "dL_bp": -0.2199082374572754, "ratio": 0.19277897398201574}}, {"step": 100, "epoch": 0, "train_loss": 0.19819821417331696, "cos_r_negg": [0.4562336206436157, 0.3609468638896942, 0.3217041492462158], "cos_innovation_negg": [0.4562336206436157, 0.3609468638896942, 0.3217041492462158], "cos_apical_negg": [0.0924934595823288, 0.07498198002576828, 0.10977184027433395], "cos_Ac_negg": [0.4592011868953705, 0.3636699914932251, 0.32186806201934814], "r_norm": [0.9399021863937378, 0.9428062438964844, 0.9601663947105408]}, {"step": 200, "epoch": 0, "train_loss": 0.32231855392456055, "cos_r_negg": [0.4961671829223633, 0.42615118622779846, 0.3737158179283142], "cos_innovation_negg": [0.4961671829223633, 0.42615118622779846, 0.3737158179283142], "cos_apical_negg": [0.08375506103038788, 0.07275334745645523, 0.10912351310253143], "cos_Ac_negg": [0.5020463466644287, 0.4308222830295563, 0.3757030963897705], "r_norm": [0.7590075135231018, 0.7669145464897156, 0.7767611742019653]}, {"step": 300, "epoch": 0, "train_loss": 0.1517620086669922, "cos_r_negg": [0.4997817575931549, 0.45438945293426514, 0.418070912361145], "cos_innovation_negg": [0.4997817575931549, 0.45438945293426514, 0.418070912361145], "cos_apical_negg": [0.06755456328392029, 0.070278100669384, 0.11749660968780518], "cos_Ac_negg": [0.5018380880355835, 0.45461800694465637, 0.41634446382522583], "r_norm": [0.5646820068359375, 0.5667244791984558, 0.57851642370224]}, {"step": 400, "epoch": 0, "train_loss": 0.12696349620819092, "cos_r_negg": [0.5265330076217651, 0.47366011142730713, 0.4304295778274536], "cos_innovation_negg": [0.5265330076217651, 0.47366011142730713, 0.4304295778274536], "cos_apical_negg": [0.05604200437664986, 0.07228812575340271, 0.10936453193426132], "cos_Ac_negg": [0.5275131464004517, 0.4741481840610504, 0.42993783950805664], "r_norm": [0.5426884889602661, 0.5449879169464111, 0.5514365434646606]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9475, "test_loss": 0.18153308029174806}, {"step": 500, "epoch": 1, "train_loss": 0.1655046045780182, "cos_r_negg": [0.5191988945007324, 0.48418760299682617, 0.4616186022758484], "cos_innovation_negg": [0.5191988945007324, 0.48418760299682617, 0.4616186022758484], "cos_apical_negg": [0.06751049309968948, 0.07074972242116928, 0.125736802816391], "cos_Ac_negg": [0.5299745798110962, 0.4899352490901947, 0.46142110228538513], "r_norm": [0.4868793189525604, 0.48760986328125, 0.48948433995246887], "ldr": {"dL_sdil": -0.01239427924156189, "dL_bp": -0.010387584567070007, "ratio": 1.1931820300990246}}, {"step": 600, "epoch": 1, "train_loss": 0.10512755811214447, "cos_r_negg": [0.506693959236145, 0.47643205523490906, 0.45370903611183167], "cos_innovation_negg": [0.506693959236145, 0.47643205523490906, 0.45370903611183167], "cos_apical_negg": [0.06160319596529007, 0.0535128079354763, 0.1167837530374527], "cos_Ac_negg": [0.5124704837799072, 0.47943711280822754, 0.4548422396183014], "r_norm": [0.47535842657089233, 0.47875380516052246, 0.4810010492801666]}, {"step": 700, "epoch": 1, "train_loss": 0.18744073808193207, "cos_r_negg": [0.5276387929916382, 0.4997781217098236, 0.48154783248901367], "cos_innovation_negg": [0.5276387929916382, 0.4997781217098236, 0.48154783248901367], "cos_apical_negg": [0.05852305144071579, 0.05880728363990784, 0.09995196014642715], "cos_Ac_negg": [0.5367352962493896, 0.5038727521896362, 0.4854159355163574], "r_norm": [0.4378526508808136, 0.4416790306568146, 0.44054216146469116]}, {"step": 800, "epoch": 1, "train_loss": 0.14523780345916748, "cos_r_negg": [0.535984456539154, 0.5065294504165649, 0.47763127088546753], "cos_innovation_negg": [0.535984456539154, 0.5065294504165649, 0.47763127088546753], "cos_apical_negg": [0.043083205819129944, 0.060550667345523834, 0.11668789386749268], "cos_Ac_negg": [0.5489453673362732, 0.5140861868858337, 0.4792100787162781], "r_norm": [0.40186405181884766, 0.4020804166793823, 0.4056309759616852]}, {"step": 900, "epoch": 1, "train_loss": 0.2052808701992035, "cos_r_negg": [0.5632014274597168, 0.5317338705062866, 0.48478347063064575], "cos_innovation_negg": [0.5632014274597168, 0.5317338705062866, 0.48478347063064575], "cos_apical_negg": [0.0345851331949234, 0.0543665736913681, 0.11113052070140839], "cos_Ac_negg": [0.5638846158981323, 0.5336527824401855, 0.48559439182281494], "r_norm": [0.3906911313533783, 0.3913891017436981, 0.393975168466568]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9574, "test_loss": 0.14721737022399903}, {"step": 1000, "epoch": 2, "train_loss": 0.024760780856013298, "cos_r_negg": [0.5648545622825623, 0.5417273640632629, 0.496168315410614], "cos_innovation_negg": [0.5648545622825623, 0.5417273640632629, 0.496168315410614], "cos_apical_negg": [0.04227800667285919, 0.05153029412031174, 0.10890823602676392], "cos_Ac_negg": [0.571323812007904, 0.545744001865387, 0.4980310797691345], "r_norm": [0.3385966420173645, 0.3391200304031372, 0.34419602155685425], "ldr": {"dL_sdil": -0.0067184194922447205, "dL_bp": -0.005749911069869995, "ratio": 1.1684388524632647}}, {"step": 1100, "epoch": 2, "train_loss": 0.16186027228832245, "cos_r_negg": [0.5603777766227722, 0.531798779964447, 0.49178510904312134], "cos_innovation_negg": [0.5603777766227722, 0.531798779964447, 0.49178510904312134], "cos_apical_negg": [0.03889578580856323, 0.047157835215330124, 0.1025448739528656], "cos_Ac_negg": [0.5667866468429565, 0.5373232364654541, 0.4949599504470825], "r_norm": [0.35095852613449097, 0.35477763414382935, 0.35318225622177124]}, {"step": 1200, "epoch": 2, "train_loss": 0.1569242626428604, "cos_r_negg": [0.5550088882446289, 0.5344139337539673, 0.4981812834739685], "cos_innovation_negg": [0.5550088882446289, 0.5344139337539673, 0.4981812834739685], "cos_apical_negg": [0.02155444025993347, 0.040098030120134354, 0.10366980731487274], "cos_Ac_negg": [0.566131591796875, 0.5400733947753906, 0.5006672143936157], "r_norm": [0.2950282692909241, 0.29617446660995483, 0.29916858673095703]}, {"step": 1300, "epoch": 2, "train_loss": 0.1459813266992569, "cos_r_negg": [0.5581264495849609, 0.5500150918960571, 0.5197196006774902], "cos_innovation_negg": [0.5581264495849609, 0.5500150918960571, 0.5197196006774902], "cos_apical_negg": [0.024192284792661667, 0.0479893758893013, 0.116562619805336], "cos_Ac_negg": [0.5768191814422607, 0.5619202852249146, 0.5239880681037903], "r_norm": [0.3309983015060425, 0.32787635922431946, 0.32926201820373535]}, {"step": 1400, "epoch": 2, "train_loss": 0.10320207476615906, "cos_r_negg": [0.5416789054870605, 0.5423123836517334, 0.5176808834075928], "cos_innovation_negg": [0.5416789054870605, 0.5423123836517334, 0.5176808834075928], "cos_apical_negg": [0.014692528173327446, 0.04524938762187958, 0.10794402658939362], "cos_Ac_negg": [0.5685117244720459, 0.5620934963226318, 0.5263524651527405], "r_norm": [0.30710625648498535, 0.30455282330513, 0.30817869305610657]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9644, "test_loss": 0.1145642152786255}, {"step": 1500, "epoch": 3, "train_loss": 0.06334341317415237, "cos_r_negg": [0.5452038049697876, 0.5396445989608765, 0.5256383419036865], "cos_innovation_negg": [0.5452038049697876, 0.5396445989608765, 0.5256383419036865], "cos_apical_negg": [0.017272096127271652, 0.03823789209127426, 0.11339951306581497], "cos_Ac_negg": [0.5736880898475647, 0.5632601976394653, 0.5382489562034607], "r_norm": [0.2519826292991638, 0.2503807246685028, 0.2527422606945038], "ldr": {"dL_sdil": -0.005224883556365967, "dL_bp": -0.005164317786693573, "ratio": 1.0117277387205814}}, {"step": 1600, "epoch": 3, "train_loss": 0.08027355372905731, "cos_r_negg": [0.5536803007125854, 0.547955334186554, 0.5315324068069458], "cos_innovation_negg": [0.5536803007125854, 0.547955334186554, 0.5315324068069458], "cos_apical_negg": [0.016805946826934814, 0.04154978692531586, 0.11257649213075638], "cos_Ac_negg": [0.5790594816207886, 0.5665426850318909, 0.5419608354568481], "r_norm": [0.27155372500419617, 0.27073079347610474, 0.27323296666145325]}, {"step": 1700, "epoch": 3, "train_loss": 0.08025884628295898, "cos_r_negg": [0.5233545303344727, 0.5304712653160095, 0.5180680751800537], "cos_innovation_negg": [0.5233545303344727, 0.5304712653160095, 0.5180680751800537], "cos_apical_negg": [0.004102821461856365, 0.037973955273628235, 0.10750962793827057], "cos_Ac_negg": [0.5699038505554199, 0.5700200796127319, 0.5456762313842773], "r_norm": [0.25968384742736816, 0.2586820125579834, 0.2624744176864624]}, {"step": 1800, "epoch": 3, "train_loss": 0.04806072264909744, "cos_r_negg": [0.5499836802482605, 0.5529208183288574, 0.549586832523346], "cos_innovation_negg": [0.5499836802482605, 0.5529208183288574, 0.549586832523346], "cos_apical_negg": [0.003273674054071307, 0.036957550793886185, 0.10541849583387375], "cos_Ac_negg": [0.5759398937225342, 0.570105791091919, 0.55732661485672], "r_norm": [0.2637656331062317, 0.2634180188179016, 0.26708728075027466]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9673, "test_loss": 0.11477804679870605}, {"step": 1900, "epoch": 4, "train_loss": 0.02444826439023018, "cos_r_negg": [0.5487892627716064, 0.5355829000473022, 0.542591392993927], "cos_innovation_negg": [0.5487892627716064, 0.5355829000473022, 0.542591392993927], "cos_apical_negg": [0.022186705842614174, 0.04220614582300186, 0.10831473767757416], "cos_Ac_negg": [0.5984379649162292, 0.5697867274284363, 0.5640909671783447], "r_norm": [0.32423821091651917, 0.32326072454452515, 0.3276006579399109]}, {"step": 2000, "epoch": 4, "train_loss": 0.12103118747472763, "cos_r_negg": [0.5188897848129272, 0.5130773186683655, 0.5285385847091675], "cos_innovation_negg": [0.5188897848129272, 0.5130773186683655, 0.5285385847091675], "cos_apical_negg": [0.003930566366761923, 0.03765185922384262, 0.09973377734422684], "cos_Ac_negg": [0.5726943016052246, 0.561560869216919, 0.5648659467697144], "r_norm": [0.23365336656570435, 0.2309618890285492, 0.2362920045852661], "ldr": {"dL_sdil": -0.00528167188167572, "dL_bp": -0.005683548748493195, "ratio": 0.9292912079052689}}, {"step": 2100, "epoch": 4, "train_loss": 0.024333203211426735, "cos_r_negg": [0.532158613204956, 0.5350310802459717, 0.5475729703903198], "cos_innovation_negg": [0.532158613204956, 0.5350310802459717, 0.5475729703903198], "cos_apical_negg": [0.012790195643901825, 0.05143173038959503, 0.10263633728027344], "cos_Ac_negg": [0.5851036310195923, 0.5792784094810486, 0.5785484313964844], "r_norm": [0.3119562864303589, 0.30925729870796204, 0.3135855495929718]}, {"step": 2200, "epoch": 4, "train_loss": 0.007553962990641594, "cos_r_negg": [0.5498648881912231, 0.5508173108100891, 0.5553864240646362], "cos_innovation_negg": [0.5498648881912231, 0.5508173108100891, 0.5553864240646362], "cos_apical_negg": [0.00876094214618206, 0.040761008858680725, 0.1037021279335022], "cos_Ac_negg": [0.6020341515541077, 0.5934842824935913, 0.5816723704338074], "r_norm": [0.23748338222503662, 0.23687006533145905, 0.23978067934513092]}, {"step": 2300, "epoch": 4, "train_loss": 0.1032857894897461, "cos_r_negg": [0.5031359195709229, 0.5203165411949158, 0.537436306476593], "cos_innovation_negg": [0.5031359195709229, 0.5203165411949158, 0.537436306476593], "cos_apical_negg": [0.015608749352395535, 0.03436913341283798, 0.10463280230760574], "cos_Ac_negg": [0.5825303196907043, 0.5863634347915649, 0.5832058787345886], "r_norm": [0.23429690301418304, 0.2324206829071045, 0.23625370860099792]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9693, "test_loss": 0.11051450176239014}, {"step": 2400, "epoch": 5, "train_loss": 0.06244773417711258, "cos_r_negg": [0.5216194987297058, 0.532431960105896, 0.5434267520904541], "cos_innovation_negg": [0.5216194987297058, 0.532431960105896, 0.5434267520904541], "cos_apical_negg": [-0.010487285442650318, 0.0381862111389637, 0.09908141940832138], "cos_Ac_negg": [0.6037192344665527, 0.5965788960456848, 0.5930641889572144], "r_norm": [0.23071017861366272, 0.22901231050491333, 0.23396210372447968]}, {"step": 2500, "epoch": 5, "train_loss": 0.01840227097272873, "cos_r_negg": [0.4826870262622833, 0.501519501209259, 0.5196942090988159], "cos_innovation_negg": [0.4826870262622833, 0.501519501209259, 0.5196942090988159], "cos_apical_negg": [0.008614158257842064, 0.04547841101884842, 0.0974075198173523], "cos_Ac_negg": [0.5942498445510864, 0.5917768478393555, 0.599340558052063], "r_norm": [0.24934172630310059, 0.24767103791236877, 0.24996879696846008], "ldr": {"dL_sdil": -0.011170715093612671, "dL_bp": -0.01398564875125885, "ratio": 0.7987269873774854}}, {"step": 2600, "epoch": 5, "train_loss": 0.061060830950737, "cos_r_negg": [0.49484387040138245, 0.5180107355117798, 0.5349160432815552], "cos_innovation_negg": [0.49484387040138245, 0.5180107355117798, 0.5349160432815552], "cos_apical_negg": [0.007128430530428886, 0.04086736589670181, 0.09839516878128052], "cos_Ac_negg": [0.598257303237915, 0.6041316986083984, 0.5988324284553528], "r_norm": [0.21950285136699677, 0.21728940308094025, 0.22030556201934814]}, {"step": 2700, "epoch": 5, "train_loss": 0.012292544357478619, "cos_r_negg": [0.49846529960632324, 0.5182821750640869, 0.5381145477294922], "cos_innovation_negg": [0.49846529960632324, 0.5182821750640869, 0.5381145477294922], "cos_apical_negg": [0.010728912428021431, 0.03891203925013542, 0.09855789691209793], "cos_Ac_negg": [0.611120343208313, 0.6131026148796082, 0.6143841743469238], "r_norm": [0.21447685360908508, 0.2124914973974228, 0.21530643105506897]}, {"step": 2800, "epoch": 5, "train_loss": 0.08087609708309174, "cos_r_negg": [0.4886190891265869, 0.5083039402961731, 0.5329645872116089], "cos_innovation_negg": [0.4886190891265869, 0.5083039402961731, 0.5329645872116089], "cos_apical_negg": [0.014900158159434795, 0.035297930240631104, 0.10018932074308395], "cos_Ac_negg": [0.611050009727478, 0.608727216720581, 0.6146820783615112], "r_norm": [0.2476438730955124, 0.24570077657699585, 0.24911633133888245]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9683, "test_loss": 0.11789603614807129}, {"step": 2900, "epoch": 6, "train_loss": 0.017274226993322372, "cos_r_negg": [0.47784823179244995, 0.4935106039047241, 0.5223339200019836], "cos_innovation_negg": [0.47784823179244995, 0.4935106039047241, 0.5223339200019836], "cos_apical_negg": [0.020545009523630142, 0.03409266471862793, 0.09988909214735031], "cos_Ac_negg": [0.6047525405883789, 0.6058627367019653, 0.6138401627540588], "r_norm": [0.2350500077009201, 0.23234309256076813, 0.23486503958702087]}, {"step": 3000, "epoch": 6, "train_loss": 0.07949283719062805, "cos_r_negg": [0.45194587111473083, 0.4786764681339264, 0.5070891380310059], "cos_innovation_negg": [0.45194587111473083, 0.4786764681339264, 0.5070891380310059], "cos_apical_negg": [0.000577261671423912, 0.03616706654429436, 0.09920153021812439], "cos_Ac_negg": [0.596362829208374, 0.6020251512527466, 0.6144667863845825], "r_norm": [0.2332625836133957, 0.23132850229740143, 0.23530133068561554], "ldr": {"dL_sdil": -0.008470743894577026, "dL_bp": -0.012227363884449005, "ratio": 0.6927694288505047}}, {"step": 3100, "epoch": 6, "train_loss": 0.011230203323066235, "cos_r_negg": [0.43944498896598816, 0.4663470983505249, 0.5067719221115112], "cos_innovation_negg": [0.43944498896598816, 0.4663470983505249, 0.5067719221115112], "cos_apical_negg": [0.0059204502031207085, 0.0318872407078743, 0.11310937255620956], "cos_Ac_negg": [0.6001513600349426, 0.6119512319564819, 0.6258364915847778], "r_norm": [0.23979894816875458, 0.23688717186450958, 0.24084894359111786]}, {"step": 3200, "epoch": 6, "train_loss": 0.04621240496635437, "cos_r_negg": [0.4342973232269287, 0.46508535742759705, 0.5044590830802917], "cos_innovation_negg": [0.4342973232269287, 0.46508535742759705, 0.5044590830802917], "cos_apical_negg": [0.0012547691585496068, 0.03494802117347717, 0.10854874551296234], "cos_Ac_negg": [0.5974046587944031, 0.6151109337806702, 0.6297565698623657], "r_norm": [0.2083405703306198, 0.20680619776248932, 0.20945681631565094]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9686, "test_loss": 0.1256407434463501}, {"step": 3300, "epoch": 7, "train_loss": 0.014249805361032486, "cos_r_negg": [0.41731202602386475, 0.4580508768558502, 0.4931107461452484], "cos_innovation_negg": [0.41731202602386475, 0.4580508768558502, 0.4931107461452484], "cos_apical_negg": [0.009109636768698692, 0.03295160457491875, 0.10847292840480804], "cos_Ac_negg": [0.5967949628829956, 0.614556074142456, 0.6270172595977783], "r_norm": [0.2500160336494446, 0.24679669737815857, 0.2498304545879364]}, {"step": 3400, "epoch": 7, "train_loss": 0.01112859882414341, "cos_r_negg": [0.38858509063720703, 0.44096070528030396, 0.480546772480011], "cos_innovation_negg": [0.38858509063720703, 0.44096070528030396, 0.480546772480011], "cos_apical_negg": [-0.008255871012806892, 0.03822627291083336, 0.11014938354492188], "cos_Ac_negg": [0.6006128787994385, 0.6202166080474854, 0.6348124742507935], "r_norm": [0.2412058562040329, 0.23865504562854767, 0.2412518858909607]}, {"step": 3500, "epoch": 7, "train_loss": 0.008623255416750908, "cos_r_negg": [0.3743264675140381, 0.4105738401412964, 0.4597768485546112], "cos_innovation_negg": [0.3743264675140381, 0.4105738401412964, 0.4597768485546112], "cos_apical_negg": [0.0028895470313727856, 0.03591875731945038, 0.11270925402641296], "cos_Ac_negg": [0.6082568764686584, 0.6242395639419556, 0.6397649049758911], "r_norm": [0.21341930329799652, 0.21139098703861237, 0.21424329280853271], "ldr": {"dL_sdil": -0.009350374341011047, "dL_bp": -0.0139445960521698, "ratio": 0.6705374831963035}}, {"step": 3600, "epoch": 7, "train_loss": 0.001493149669840932, "cos_r_negg": [0.3852287530899048, 0.42796075344085693, 0.47940123081207275], "cos_innovation_negg": [0.3852287530899048, 0.42796075344085693, 0.47940123081207275], "cos_apical_negg": [-0.00488023366779089, 0.034974534064531326, 0.1126023679971695], "cos_Ac_negg": [0.6145345568656921, 0.6327401995658875, 0.649524986743927], "r_norm": [0.20902538299560547, 0.20602154731750488, 0.2091146558523178]}, {"step": 3700, "epoch": 7, "train_loss": 0.005869025364518166, "cos_r_negg": [0.38824576139450073, 0.4320777654647827, 0.4697416424751282], "cos_innovation_negg": [0.38824576139450073, 0.4320777654647827, 0.4697416424751282], "cos_apical_negg": [-0.006503776181489229, 0.028564482927322388, 0.10319067537784576], "cos_Ac_negg": [0.6190537214279175, 0.6311739087104797, 0.6462790369987488], "r_norm": [0.2119978666305542, 0.20814868807792664, 0.2123369574546814]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9701, "test_loss": 0.12263582782745361}, {"step": 3800, "epoch": 8, "train_loss": 0.0028725494630634785, "cos_r_negg": [0.3476846218109131, 0.3942021131515503, 0.4406149983406067], "cos_innovation_negg": [0.3476846218109131, 0.3942021131515503, 0.4406149983406067], "cos_apical_negg": [-0.003340412862598896, 0.032182492315769196, 0.10314054787158966], "cos_Ac_negg": [0.6167541742324829, 0.6279117465019226, 0.6491929292678833], "r_norm": [0.21939122676849365, 0.21579807996749878, 0.21942919492721558]}, {"step": 3900, "epoch": 8, "train_loss": 0.01645852066576481, "cos_r_negg": [0.3188874423503876, 0.3711095452308655, 0.41697824001312256], "cos_innovation_negg": [0.3188874423503876, 0.3711095452308655, 0.41697824001312256], "cos_apical_negg": [-0.0089386235922575, 0.030950363725423813, 0.10144755244255066], "cos_Ac_negg": [0.6055940985679626, 0.6227110624313354, 0.6511268615722656], "r_norm": [0.225270614027977, 0.22113573551177979, 0.2254658341407776]}, {"step": 4000, "epoch": 8, "train_loss": 0.01775703765451908, "cos_r_negg": [0.33081763982772827, 0.3760502338409424, 0.42436355352401733], "cos_innovation_negg": [0.33081763982772827, 0.3760502338409424, 0.42436355352401733], "cos_apical_negg": [0.004012702964246273, 0.03218439593911171, 0.10053519904613495], "cos_Ac_negg": [0.6183075904846191, 0.6272047758102417, 0.6522946357727051], "r_norm": [0.2212454229593277, 0.21894893050193787, 0.22263962030410767], "ldr": {"dL_sdil": -0.010140083730220795, "dL_bp": -0.017603881657123566, "ratio": 0.5760140818782158}}, {"step": 4100, "epoch": 8, "train_loss": 0.013629444874823093, "cos_r_negg": [0.3223866820335388, 0.37011513113975525, 0.4229600131511688], "cos_innovation_negg": [0.3223866820335388, 0.37011513113975525, 0.4229600131511688], "cos_apical_negg": [0.0003302227705717087, 0.03650972247123718, 0.10796025395393372], "cos_Ac_negg": [0.6143237352371216, 0.6272937059402466, 0.6533287763595581], "r_norm": [0.22584208846092224, 0.22285646200180054, 0.22673724591732025]}, {"step": 4200, "epoch": 8, "train_loss": 0.012107991613447666, "cos_r_negg": [0.28532353043556213, 0.34739142656326294, 0.39834463596343994], "cos_innovation_negg": [0.28532353043556213, 0.34739142656326294, 0.39834463596343994], "cos_apical_negg": [-0.008128422312438488, 0.037260595709085464, 0.10063967853784561], "cos_Ac_negg": [0.6082413196563721, 0.6286255121231079, 0.653980553150177], "r_norm": [0.21755939722061157, 0.21517139673233032, 0.21883633732795715]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9712, "test_loss": 0.13053375539779663}, {"step": 4300, "epoch": 9, "train_loss": 0.006148484535515308, "cos_r_negg": [0.3049679100513458, 0.35393524169921875, 0.4092366099357605], "cos_innovation_negg": [0.3049679100513458, 0.35393524169921875, 0.4092366099357605], "cos_apical_negg": [-0.0028543826192617416, 0.038646213710308075, 0.1020691990852356], "cos_Ac_negg": [0.6200163960456848, 0.6370707750320435, 0.6626262068748474], "r_norm": [0.22027118504047394, 0.21733739972114563, 0.22126659750938416]}, {"step": 4400, "epoch": 9, "train_loss": 0.0008114433148875833, "cos_r_negg": [0.2994518280029297, 0.3576602637767792, 0.4081227481365204], "cos_innovation_negg": [0.2994518280029297, 0.3576602637767792, 0.4081227481365204], "cos_apical_negg": [-0.008132841438055038, 0.039271071553230286, 0.10279826819896698], "cos_Ac_negg": [0.6209456324577332, 0.6397671103477478, 0.6645205616950989], "r_norm": [0.22258585691452026, 0.22009657323360443, 0.223506361246109]}, {"step": 4500, "epoch": 9, "train_loss": 0.022984560579061508, "cos_r_negg": [0.26498422026634216, 0.32743608951568604, 0.3742830753326416], "cos_innovation_negg": [0.26498422026634216, 0.32743608951568604, 0.3742830753326416], "cos_apical_negg": [-0.006952760741114616, 0.03886932134628296, 0.10099170356988907], "cos_Ac_negg": [0.6116693019866943, 0.6377890110015869, 0.6633337736129761], "r_norm": [0.23831143975257874, 0.23460842669010162, 0.23828981816768646], "ldr": {"dL_sdil": -0.011285528540611267, "dL_bp": -0.020764201879501343, "ratio": 0.5435089008526964}}, {"step": 4600, "epoch": 9, "train_loss": 0.0007069878047332168, "cos_r_negg": [0.2655528485774994, 0.3202352523803711, 0.3710695505142212], "cos_innovation_negg": [0.2655528485774994, 0.3202352523803711, 0.3710695505142212], "cos_apical_negg": [0.0013217898085713387, 0.03834256902337074, 0.10062165558338165], "cos_Ac_negg": [0.6067532300949097, 0.6318719387054443, 0.6621993780136108], "r_norm": [0.2354167103767395, 0.2314644306898117, 0.23491345345973969]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9727, "test_loss": 0.12967014846801758}, {"step": 4700, "epoch": 10, "train_loss": 0.0023708001244813204, "cos_r_negg": [0.24326619505882263, 0.2900390625, 0.34820228815078735], "cos_innovation_negg": [0.24326619505882263, 0.2900390625, 0.34820228815078735], "cos_apical_negg": [-3.511086106300354e-06, 0.032490044832229614, 0.0980047732591629], "cos_Ac_negg": [0.6065478324890137, 0.6289293169975281, 0.6626633405685425], "r_norm": [0.2016417682170868, 0.19869792461395264, 0.20257467031478882]}, {"step": 4800, "epoch": 10, "train_loss": 0.0020702225156128407, "cos_r_negg": [0.24571841955184937, 0.2944677174091339, 0.3514004349708557], "cos_innovation_negg": [0.24571841955184937, 0.2944677174091339, 0.3514004349708557], "cos_apical_negg": [-0.002526048105210066, 0.03226470202207565, 0.09737521409988403], "cos_Ac_negg": [0.6093655228614807, 0.6314992904663086, 0.6656678318977356], "r_norm": [0.20281454920768738, 0.19931988418102264, 0.20342528820037842]}, {"step": 4900, "epoch": 10, "train_loss": 0.0019158375216647983, "cos_r_negg": [0.24141237139701843, 0.28968027234077454, 0.34494662284851074], "cos_innovation_negg": [0.24141237139701843, 0.28968027234077454, 0.34494662284851074], "cos_apical_negg": [-0.0023834090679883957, 0.033177558332681656, 0.09725464880466461], "cos_Ac_negg": [0.6087672114372253, 0.6332275867462158, 0.6685798168182373], "r_norm": [0.20675504207611084, 0.20340441167354584, 0.20745927095413208]}, {"step": 5000, "epoch": 10, "train_loss": 0.0008548484183847904, "cos_r_negg": [0.23028664290905, 0.28492194414138794, 0.33767589926719666], "cos_innovation_negg": [0.23028664290905, 0.28492194414138794, 0.33767589926719666], "cos_apical_negg": [-0.0038913972675800323, 0.03447958081960678, 0.09226565808057785], "cos_Ac_negg": [0.6001114845275879, 0.6293758749961853, 0.6673866510391235], "r_norm": [0.20831169188022614, 0.2051803320646286, 0.20866748690605164], "ldr": {"dL_sdil": -0.012155190110206604, "dL_bp": -0.0229220911860466, "ratio": 0.5302827744444997}}, {"step": 5100, "epoch": 10, "train_loss": 0.0010202466510236263, "cos_r_negg": [0.2341373860836029, 0.28416383266448975, 0.340952068567276], "cos_innovation_negg": [0.2341373860836029, 0.28416383266448975, 0.340952068567276], "cos_apical_negg": [-0.00420326367020607, 0.033878736197948456, 0.09707412123680115], "cos_Ac_negg": [0.607232391834259, 0.6333211660385132, 0.6719170212745667], "r_norm": [0.20714673399925232, 0.2038562297821045, 0.2072623074054718]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9724, "test_loss": 0.13474912872314454}, {"step": 5200, "epoch": 11, "train_loss": 0.0006876580300740898, "cos_r_negg": [0.21897663176059723, 0.27664655447006226, 0.33035069704055786], "cos_innovation_negg": [0.21897663176059723, 0.27664655447006226, 0.33035069704055786], "cos_apical_negg": [-0.005986626259982586, 0.03328913077712059, 0.09508834779262543], "cos_Ac_negg": [0.5988229513168335, 0.6279399394989014, 0.672398567199707], "r_norm": [0.20531290769577026, 0.20248490571975708, 0.20603346824645996]}, {"step": 5300, "epoch": 11, "train_loss": 0.0012513920664787292, "cos_r_negg": [0.2129746377468109, 0.26975369453430176, 0.3232852816581726], "cos_innovation_negg": [0.2129746377468109, 0.26975369453430176, 0.3232852816581726], "cos_apical_negg": [-0.005739088170230389, 0.03182534873485565, 0.09561654925346375], "cos_Ac_negg": [0.5997483730316162, 0.6285946369171143, 0.6708213090896606], "r_norm": [0.20247995853424072, 0.19919392466545105, 0.20331595838069916]}, {"step": 5400, "epoch": 11, "train_loss": 0.002956363372504711, "cos_r_negg": [0.2116379737854004, 0.2628961205482483, 0.31698405742645264], "cos_innovation_negg": [0.2116379737854004, 0.2628961205482483, 0.31698405742645264], "cos_apical_negg": [-0.0015459386631846428, 0.03349905461072922, 0.09399515390396118], "cos_Ac_negg": [0.6028150916099548, 0.6296771168708801, 0.6726018190383911], "r_norm": [0.2075035572052002, 0.20453691482543945, 0.20845350623130798]}, {"step": 5500, "epoch": 11, "train_loss": 0.0026475482154637575, "cos_r_negg": [0.20966067910194397, 0.26422518491744995, 0.31851285696029663], "cos_innovation_negg": [0.20966067910194397, 0.26422518491744995, 0.31851285696029663], "cos_apical_negg": [-0.0028996989130973816, 0.03485501557588577, 0.09448803961277008], "cos_Ac_negg": [0.6050945520401001, 0.6289932727813721, 0.6731719970703125], "r_norm": [0.21088801324367523, 0.20764297246932983, 0.2111128866672516], "ldr": {"dL_sdil": -0.011994272470474243, "dL_bp": -0.023104965686798096, "ratio": 0.5191209817432288}}, {"step": 5600, "epoch": 11, "train_loss": 0.00218569696880877, "cos_r_negg": [0.19715338945388794, 0.25238174200057983, 0.30352944135665894], "cos_innovation_negg": [0.19715338945388794, 0.25238174200057983, 0.30352944135665894], "cos_apical_negg": [-0.004069292917847633, 0.033313825726509094, 0.09001317620277405], "cos_Ac_negg": [0.6001343727111816, 0.6257193088531494, 0.6720417141914368], "r_norm": [0.2028658092021942, 0.19982746243476868, 0.20375166833400726]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9724, "test_loss": 0.13916511611938476}, {"step": 5700, "epoch": 12, "train_loss": 0.0020794891752302647, "cos_r_negg": [0.20272225141525269, 0.2574833333492279, 0.307792991399765], "cos_innovation_negg": [0.20272225141525269, 0.2574833333492279, 0.307792991399765], "cos_apical_negg": [-0.002199282869696617, 0.03414363041520119, 0.09020665287971497], "cos_Ac_negg": [0.6050809621810913, 0.6289219260215759, 0.6739180684089661], "r_norm": [0.20607653260231018, 0.2031911015510559, 0.20651839673519135]}, {"step": 5800, "epoch": 12, "train_loss": 0.00020566982857417315, "cos_r_negg": [0.19398340582847595, 0.2503836154937744, 0.3005009889602661], "cos_innovation_negg": [0.19398340582847595, 0.2503836154937744, 0.3005009889602661], "cos_apical_negg": [-0.004616866819560528, 0.03416449576616287, 0.08954647183418274], "cos_Ac_negg": [0.6008589267730713, 0.6273083686828613, 0.6730796694755554], "r_norm": [0.20475776493549347, 0.20183506608009338, 0.20556774735450745]}, {"step": 5900, "epoch": 12, "train_loss": 0.0018817087402567267, "cos_r_negg": [0.19192993640899658, 0.24909460544586182, 0.29923221468925476], "cos_innovation_negg": [0.19192993640899658, 0.24909460544586182, 0.29923221468925476], "cos_apical_negg": [-0.004756544716656208, 0.0348251648247242, 0.08955187350511551], "cos_Ac_negg": [0.5998852252960205, 0.6275185346603394, 0.6741136312484741], "r_norm": [0.20769557356834412, 0.2047315537929535, 0.20835444331169128]}, {"step": 6000, "epoch": 12, "train_loss": 0.0005604384932667017, "cos_r_negg": [0.18731625378131866, 0.2441646009683609, 0.2948216199874878], "cos_innovation_negg": [0.18731625378131866, 0.2441646009683609, 0.2948216199874878], "cos_apical_negg": [-0.00470914039760828, 0.03447927534580231, 0.08883833885192871], "cos_Ac_negg": [0.599158525466919, 0.6261067390441895, 0.6752423048019409], "r_norm": [0.20639361441135406, 0.2033909112215042, 0.20716290175914764], "ldr": {"dL_sdil": -0.01247800886631012, "dL_bp": -0.02522388845682144, "ratio": 0.49469013818666885}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9723, "test_loss": 0.14235655269622802}, {"step": 6100, "epoch": 13, "train_loss": 0.0007953558815643191, "cos_r_negg": [0.18446385860443115, 0.2396831214427948, 0.290547251701355], "cos_innovation_negg": [0.18446385860443115, 0.2396831214427948, 0.290547251701355], "cos_apical_negg": [-0.002962573431432247, 0.033410150557756424, 0.08906933665275574], "cos_Ac_negg": [0.5952847599983215, 0.6237782835960388, 0.674699068069458], "r_norm": [0.20510908961296082, 0.20224229991436005, 0.2059614062309265]}, {"step": 6200, "epoch": 13, "train_loss": 0.0007006739033386111, "cos_r_negg": [0.17903070151805878, 0.2364688217639923, 0.28458109498023987], "cos_innovation_negg": [0.17903070151805878, 0.2364688217639923, 0.28458109498023987], "cos_apical_negg": [-0.0029544783756136894, 0.033886633813381195, 0.0861949771642685], "cos_Ac_negg": [0.5941613912582397, 0.6231882572174072, 0.6737856268882751], "r_norm": [0.20340999960899353, 0.20053435862064362, 0.20420712232589722]}, {"step": 6300, "epoch": 13, "train_loss": 0.0003973285492975265, "cos_r_negg": [0.17568711936473846, 0.231784388422966, 0.28029865026474], "cos_innovation_negg": [0.17568711936473846, 0.231784388422966, 0.28029865026474], "cos_apical_negg": [-0.002083425410091877, 0.03384298086166382, 0.08610855042934418], "cos_Ac_negg": [0.5944377183914185, 0.6225796937942505, 0.6730598211288452], "r_norm": [0.20646843314170837, 0.20356518030166626, 0.2072640061378479]}, {"step": 6400, "epoch": 13, "train_loss": 0.0006554600549861789, "cos_r_negg": [0.17507170140743256, 0.23097488284111023, 0.2792336940765381], "cos_innovation_negg": [0.17507170140743256, 0.23097488284111023, 0.2792336940765381], "cos_apical_negg": [-0.002629481256008148, 0.03345771133899689, 0.08591315150260925], "cos_Ac_negg": [0.5959250330924988, 0.6225063800811768, 0.6728956699371338], "r_norm": [0.20461906492710114, 0.2017722874879837, 0.20540666580200195]}, {"step": 6500, "epoch": 13, "train_loss": 0.0001446559908799827, "cos_r_negg": [0.17442870140075684, 0.22965818643569946, 0.27639836072921753], "cos_innovation_negg": [0.17442870140075684, 0.22965818643569946, 0.27639836072921753], "cos_apical_negg": [-0.003029278479516506, 0.03396303951740265, 0.08407631516456604], "cos_Ac_negg": [0.5945233106613159, 0.6232552528381348, 0.6756317615509033], "r_norm": [0.20908938348293304, 0.20641925930976868, 0.20985308289527893], "ldr": {"dL_sdil": -0.013210579752922058, "dL_bp": -0.027561113238334656, "ratio": 0.4793195267071981}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9726, "test_loss": 0.1455043478012085}, {"step": 6600, "epoch": 14, "train_loss": 0.0004824562347494066, "cos_r_negg": [0.1671731024980545, 0.22329702973365784, 0.2709178328514099], "cos_innovation_negg": [0.1671731024980545, 0.22329702973365784, 0.2709178328514099], "cos_apical_negg": [-0.004154294729232788, 0.03307810425758362, 0.08348521590232849], "cos_Ac_negg": [0.5958958268165588, 0.6226828098297119, 0.6753007173538208], "r_norm": [0.20208407938480377, 0.19935184717178345, 0.20300203561782837]}, {"step": 6700, "epoch": 14, "train_loss": 0.0008160572033375502, "cos_r_negg": [0.16498219966888428, 0.22207579016685486, 0.2698427140712738], "cos_innovation_negg": [0.16498219966888428, 0.22207579016685486, 0.2698427140712738], "cos_apical_negg": [-0.0052373334765434265, 0.03249061107635498, 0.08310656994581223], "cos_Ac_negg": [0.5944553017616272, 0.6222578883171082, 0.6761384606361389], "r_norm": [0.2044653594493866, 0.20145821571350098, 0.20519861578941345]}, {"step": 6800, "epoch": 14, "train_loss": 0.0009065538179129362, "cos_r_negg": [0.1594417840242386, 0.21674184501171112, 0.2647378146648407], "cos_innovation_negg": [0.1594417840242386, 0.21674184501171112, 0.2647378146648407], "cos_apical_negg": [-0.004249837249517441, 0.032414667308330536, 0.08174674957990646], "cos_Ac_negg": [0.5946006774902344, 0.6216984987258911, 0.6755374073982239], "r_norm": [0.20176371932029724, 0.19887161254882812, 0.2025604546070099]}, {"step": 6900, "epoch": 14, "train_loss": 0.0006018164567649364, "cos_r_negg": [0.15775761008262634, 0.2153831422328949, 0.26428309082984924], "cos_innovation_negg": [0.15775761008262634, 0.2153831422328949, 0.26428309082984924], "cos_apical_negg": [-0.005284152925014496, 0.03316710144281387, 0.08256210386753082], "cos_Ac_negg": [0.5963074564933777, 0.6216413378715515, 0.6754178404808044], "r_norm": [0.20030176639556885, 0.19743712246418, 0.20103485882282257]}, {"step": 7000, "epoch": 14, "train_loss": 0.0007463142392225564, "cos_r_negg": [0.16179631650447845, 0.21716907620429993, 0.26553523540496826], "cos_innovation_negg": [0.16179631650447845, 0.21716907620429993, 0.26553523540496826], "cos_apical_negg": [-0.0037459516897797585, 0.03344787657260895, 0.08418938517570496], "cos_Ac_negg": [0.5923255681991577, 0.6215612888336182, 0.6758936047554016], "r_norm": [0.20400476455688477, 0.20118750631809235, 0.20500923693180084], "ldr": {"dL_sdil": -0.012740954756736755, "dL_bp": -0.02715706080198288, "ratio": 0.4691580892953803}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9725, "test_loss": 0.14923210258483888}], "final": {"test_acc": 0.9725, "test_loss": 0.14923210258483888, "wall_s": 29.01899290084839, "cos_r_negg": [0.16006410121917725, 0.2159397006034851, 0.26292920112609863], "cos_innovation_negg": [0.16006410121917725, 0.2159397006034851, 0.26292920112609863], "cos_apical_negg": [-0.0034941332414746284, 0.03366870433092117, 0.08231043815612793], "cos_Ac_negg": [0.5925276875495911, 0.6217751502990723, 0.6762490272521973]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_residual_s2.json b/results/nuis_ctrl_v1_rho0p5_residual_s2.json new file mode 100644 index 0000000..f797ad7 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_residual_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_residual_s2", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4534778594970703, "cos_r_negg": [0.01767568849027157, 0.03728450462222099, 0.002514135092496872], "cos_innovation_negg": [0.01767568849027157, 0.03728450462222099, 0.002514135092496872], "cos_apical_negg": [0.010833671316504478, 0.027581479400396347, -0.0016930301208049059], "cos_Ac_negg": [0.017678918316960335, 0.037285104393959045, 0.0025148179847747087], "r_norm": [4.749838829040527, 4.774457931518555, 4.856014251708984], "ldr": {"dL_sdil": -0.04106760025024414, "dL_bp": -0.24917912483215332, "ratio": 0.16481155986845694}}, {"step": 100, "epoch": 0, "train_loss": 0.17370890080928802, "cos_r_negg": [0.4979098439216614, 0.4032244086265564, 0.3694140911102295], "cos_innovation_negg": [0.4979098439216614, 0.4032244086265564, 0.3694140911102295], "cos_apical_negg": [0.07757236063480377, 0.08373922109603882, 0.13979242742061615], "cos_Ac_negg": [0.4991167485713959, 0.4050253629684448, 0.3697149157524109], "r_norm": [0.8731355667114258, 0.8851726651191711, 0.8979164958000183]}, {"step": 200, "epoch": 0, "train_loss": 0.3277958035469055, "cos_r_negg": [0.4963236451148987, 0.4413507580757141, 0.4046681821346283], "cos_innovation_negg": [0.4963236451148987, 0.4413507580757141, 0.4046681821346283], "cos_apical_negg": [0.07801713049411774, 0.08841277658939362, 0.13406099379062653], "cos_Ac_negg": [0.49723032116889954, 0.4421723484992981, 0.4038205146789551], "r_norm": [0.7200726270675659, 0.7262669801712036, 0.7435392141342163]}, {"step": 300, "epoch": 0, "train_loss": 0.1763588935136795, "cos_r_negg": [0.5336802005767822, 0.451896995306015, 0.4053812026977539], "cos_innovation_negg": [0.5336802005767822, 0.451896995306015, 0.4053812026977539], "cos_apical_negg": [0.058152634650468826, 0.07896234840154648, 0.12435396015644073], "cos_Ac_negg": [0.5322777032852173, 0.45050883293151855, 0.4030649662017822], "r_norm": [0.5215291976928711, 0.5269149541854858, 0.5367172956466675]}, {"step": 400, "epoch": 0, "train_loss": 0.1377820372581482, "cos_r_negg": [0.5447260141372681, 0.5022690892219543, 0.42625054717063904], "cos_innovation_negg": [0.5447260141372681, 0.5022690892219543, 0.42625054717063904], "cos_apical_negg": [0.060638345777988434, 0.09397780895233154, 0.114785335958004], "cos_Ac_negg": [0.5446261167526245, 0.5016587376594543, 0.4260125458240509], "r_norm": [0.489670991897583, 0.49621665477752686, 0.5032377243041992]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9467, "test_loss": 0.17786335983276366}, {"step": 500, "epoch": 1, "train_loss": 0.1921863853931427, "cos_r_negg": [0.5020118355751038, 0.4951060712337494, 0.438052773475647], "cos_innovation_negg": [0.5020118355751038, 0.4951060712337494, 0.438052773475647], "cos_apical_negg": [0.03922445327043533, 0.0842115581035614, 0.11223204433917999], "cos_Ac_negg": [0.5044324398040771, 0.49822354316711426, 0.4387269616127014], "r_norm": [0.4749787747859955, 0.48177844285964966, 0.49140626192092896], "ldr": {"dL_sdil": -0.013268277049064636, "dL_bp": -0.011191785335540771, "ratio": 1.1855371284624028}}, {"step": 600, "epoch": 1, "train_loss": 0.11350979655981064, "cos_r_negg": [0.5353504419326782, 0.5123176574707031, 0.4312099516391754], "cos_innovation_negg": [0.5353504419326782, 0.5123176574707031, 0.4312099516391754], "cos_apical_negg": [0.058454111218452454, 0.08423188328742981, 0.10041244328022003], "cos_Ac_negg": [0.5376476645469666, 0.511799693107605, 0.4314008355140686], "r_norm": [0.43762171268463135, 0.4410935044288635, 0.45054498314857483]}, {"step": 700, "epoch": 1, "train_loss": 0.1913631558418274, "cos_r_negg": [0.5469750165939331, 0.5333797931671143, 0.4553975462913513], "cos_innovation_negg": [0.5469750165939331, 0.5333797931671143, 0.4553975462913513], "cos_apical_negg": [0.051164597272872925, 0.08192162215709686, 0.11532846093177795], "cos_Ac_negg": [0.5500829219818115, 0.5355862975120544, 0.4580218195915222], "r_norm": [0.43344199657440186, 0.43722301721572876, 0.4450470805168152]}, {"step": 800, "epoch": 1, "train_loss": 0.12916748225688934, "cos_r_negg": [0.5494622588157654, 0.5454022288322449, 0.47729676961898804], "cos_innovation_negg": [0.5494622588157654, 0.5454022288322449, 0.47729676961898804], "cos_apical_negg": [0.04308928921818733, 0.06881507486104965, 0.10532616078853607], "cos_Ac_negg": [0.5515763163566589, 0.549803614616394, 0.4803372919559479], "r_norm": [0.3213401436805725, 0.3240063190460205, 0.3320201635360718]}, {"step": 900, "epoch": 1, "train_loss": 0.19544081389904022, "cos_r_negg": [0.5624649524688721, 0.5584672689437866, 0.4698528051376343], "cos_innovation_negg": [0.5624649524688721, 0.5584672689437866, 0.4698528051376343], "cos_apical_negg": [0.04868795722723007, 0.08337309956550598, 0.10268043726682663], "cos_Ac_negg": [0.5616205930709839, 0.5579421520233154, 0.4687705338001251], "r_norm": [0.4298001229763031, 0.4345526397228241, 0.44232234358787537]}, {"epoch_end": 1, "step": 938, "test_acc": 0.959, "test_loss": 0.136350793838501}, {"step": 1000, "epoch": 2, "train_loss": 0.021037617698311806, "cos_r_negg": [0.5516715049743652, 0.5676910281181335, 0.4811541438102722], "cos_innovation_negg": [0.5516715049743652, 0.5676910281181335, 0.4811541438102722], "cos_apical_negg": [0.03314455971121788, 0.07945408672094345, 0.10955093055963516], "cos_Ac_negg": [0.5559413433074951, 0.5714534521102905, 0.483421266078949], "r_norm": [0.326733261346817, 0.32843858003616333, 0.3387078046798706], "ldr": {"dL_sdil": -0.00954977422952652, "dL_bp": -0.00775078684091568, "ratio": 1.232103839975853}}, {"step": 1100, "epoch": 2, "train_loss": 0.08772651851177216, "cos_r_negg": [0.5645239353179932, 0.581345796585083, 0.5072296857833862], "cos_innovation_negg": [0.5645239353179932, 0.581345796585083, 0.5072296857833862], "cos_apical_negg": [0.03411669284105301, 0.0846252590417862, 0.10793258249759674], "cos_Ac_negg": [0.5686287879943848, 0.5849742889404297, 0.510229229927063], "r_norm": [0.3367329239845276, 0.34003746509552, 0.3485863208770752]}, {"step": 1200, "epoch": 2, "train_loss": 0.15639618039131165, "cos_r_negg": [0.5672853589057922, 0.5732043981552124, 0.5028058886528015], "cos_innovation_negg": [0.5672853589057922, 0.5732043981552124, 0.5028058886528015], "cos_apical_negg": [0.03991822898387909, 0.07897351682186127, 0.10003751516342163], "cos_Ac_negg": [0.5763939619064331, 0.5765974521636963, 0.5061700940132141], "r_norm": [0.3120023012161255, 0.3153430223464966, 0.3218724727630615]}, {"step": 1300, "epoch": 2, "train_loss": 0.12195512652397156, "cos_r_negg": [0.5721479654312134, 0.5823063254356384, 0.5008890628814697], "cos_innovation_negg": [0.5721479654312134, 0.5823063254356384, 0.5008890628814697], "cos_apical_negg": [0.04020024836063385, 0.06639070808887482, 0.09347959607839584], "cos_Ac_negg": [0.5782443881034851, 0.5875898003578186, 0.5033410787582397], "r_norm": [0.3134698271751404, 0.31710779666900635, 0.32279807329177856]}, {"step": 1400, "epoch": 2, "train_loss": 0.13875986635684967, "cos_r_negg": [0.5653266310691833, 0.5806019902229309, 0.49355965852737427], "cos_innovation_negg": [0.5653266310691833, 0.5806019902229309, 0.49355965852737427], "cos_apical_negg": [0.037827812135219574, 0.06081753224134445, 0.10684904456138611], "cos_Ac_negg": [0.5844625234603882, 0.5961549282073975, 0.4977683126926422], "r_norm": [0.2973756194114685, 0.30036216974258423, 0.30341577529907227]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9664, "test_loss": 0.11302518634796142}, {"step": 1500, "epoch": 3, "train_loss": 0.06028618663549423, "cos_r_negg": [0.5633412003517151, 0.581645131111145, 0.5090212225914001], "cos_innovation_negg": [0.5633412003517151, 0.581645131111145, 0.5090212225914001], "cos_apical_negg": [0.02568996138870716, 0.0659070760011673, 0.10494665801525116], "cos_Ac_negg": [0.5794579982757568, 0.592542827129364, 0.5132648944854736], "r_norm": [0.27191630005836487, 0.2733272910118103, 0.2775626480579376], "ldr": {"dL_sdil": -0.005582928657531738, "dL_bp": -0.0050752609968185425, "ratio": 1.1000278923648321}}, {"step": 1600, "epoch": 3, "train_loss": 0.06352798640727997, "cos_r_negg": [0.5733667612075806, 0.5936024188995361, 0.5266447067260742], "cos_innovation_negg": [0.5733667612075806, 0.5936024188995361, 0.5266447067260742], "cos_apical_negg": [0.030817529186606407, 0.06379072368144989, 0.09993778169155121], "cos_Ac_negg": [0.5871871709823608, 0.6075624227523804, 0.5335290431976318], "r_norm": [0.29693546891212463, 0.2998005151748657, 0.30487024784088135]}, {"step": 1700, "epoch": 3, "train_loss": 0.032806094735860825, "cos_r_negg": [0.5273157358169556, 0.5585248470306396, 0.5182610750198364], "cos_innovation_negg": [0.5273157358169556, 0.5585248470306396, 0.5182610750198364], "cos_apical_negg": [0.018409255892038345, 0.0573413260281086, 0.1046614944934845], "cos_Ac_negg": [0.5797021985054016, 0.5988168716430664, 0.541889488697052], "r_norm": [0.24079588055610657, 0.24336358904838562, 0.24881190061569214]}, {"step": 1800, "epoch": 3, "train_loss": 0.026628512889146805, "cos_r_negg": [0.5685943365097046, 0.604702353477478, 0.5334007740020752], "cos_innovation_negg": [0.5685943365097046, 0.604702353477478, 0.5334007740020752], "cos_apical_negg": [0.011840914376080036, 0.05348370224237442, 0.09288214147090912], "cos_Ac_negg": [0.5859178304672241, 0.6219295859336853, 0.5435793399810791], "r_norm": [0.214270681142807, 0.21516406536102295, 0.22198174893856049]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9646, "test_loss": 0.11931605224609375}, {"step": 1900, "epoch": 4, "train_loss": 0.031216194853186607, "cos_r_negg": [0.5597426891326904, 0.5946007966995239, 0.5374516844749451], "cos_innovation_negg": [0.5597426891326904, 0.5946007966995239, 0.5374516844749451], "cos_apical_negg": [0.016347669064998627, 0.05661839246749878, 0.09268146753311157], "cos_Ac_negg": [0.5795313119888306, 0.6110771894454956, 0.5477713346481323], "r_norm": [0.2539307773113251, 0.2558459937572479, 0.25997546315193176]}, {"step": 2000, "epoch": 4, "train_loss": 0.05871080234646797, "cos_r_negg": [0.5101077556610107, 0.562905490398407, 0.52896648645401], "cos_innovation_negg": [0.5101077556610107, 0.562905490398407, 0.52896648645401], "cos_apical_negg": [0.023031894117593765, 0.05405349284410477, 0.10039262473583221], "cos_Ac_negg": [0.5665547847747803, 0.6118184328079224, 0.5603225231170654], "r_norm": [0.19687366485595703, 0.19745714962482452, 0.20011885464191437], "ldr": {"dL_sdil": -0.004442073404788971, "dL_bp": -0.004386350512504578, "ratio": 1.012703702571315}}, {"step": 2100, "epoch": 4, "train_loss": 0.029522033408284187, "cos_r_negg": [0.5631446838378906, 0.6010960936546326, 0.5554468631744385], "cos_innovation_negg": [0.5631446838378906, 0.6010960936546326, 0.5554468631744385], "cos_apical_negg": [0.028458602726459503, 0.059671856462955475, 0.10500374436378479], "cos_Ac_negg": [0.5893241763114929, 0.6266671419143677, 0.5720638036727905], "r_norm": [0.21763937175273895, 0.2181074023246765, 0.22379150986671448]}, {"step": 2200, "epoch": 4, "train_loss": 0.009474407881498337, "cos_r_negg": [0.5310007929801941, 0.5935440063476562, 0.5493039488792419], "cos_innovation_negg": [0.5310007929801941, 0.5935440063476562, 0.5493039488792419], "cos_apical_negg": [0.009159318171441555, 0.05881279706954956, 0.10290274024009705], "cos_Ac_negg": [0.5896780490875244, 0.6344751715660095, 0.5688648223876953], "r_norm": [0.21013401448726654, 0.21182213723659515, 0.21715858578681946]}, {"step": 2300, "epoch": 4, "train_loss": 0.16094551980495453, "cos_r_negg": [0.5319625735282898, 0.5746119022369385, 0.5343822240829468], "cos_innovation_negg": [0.5319625735282898, 0.5746119022369385, 0.5343822240829468], "cos_apical_negg": [0.01774989627301693, 0.0619872510433197, 0.09896283596754074], "cos_Ac_negg": [0.5848884582519531, 0.6223403811454773, 0.5667043328285217], "r_norm": [0.22298488020896912, 0.2247646152973175, 0.229026198387146]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9656, "test_loss": 0.12283286952972412}, {"step": 2400, "epoch": 5, "train_loss": 0.0594419501721859, "cos_r_negg": [0.5158715844154358, 0.5717741250991821, 0.5491726994514465], "cos_innovation_negg": [0.5158715844154358, 0.5717741250991821, 0.5491726994514465], "cos_apical_negg": [0.011690962128341198, 0.058469995856285095, 0.094109907746315], "cos_Ac_negg": [0.5861676931381226, 0.6315935850143433, 0.5891934633255005], "r_norm": [0.2048872709274292, 0.20710504055023193, 0.21158148348331451]}, {"step": 2500, "epoch": 5, "train_loss": 0.025101259350776672, "cos_r_negg": [0.46053561568260193, 0.5304361581802368, 0.5122946500778198], "cos_innovation_negg": [0.46053561568260193, 0.5304361581802368, 0.5122946500778198], "cos_apical_negg": [0.01943790726363659, 0.06475730240345001, 0.111390121281147], "cos_Ac_negg": [0.5869365334510803, 0.6260303258895874, 0.5814851522445679], "r_norm": [0.25761112570762634, 0.2600604295730591, 0.2656621038913727], "ldr": {"dL_sdil": -0.011683769524097443, "dL_bp": -0.013283863663673401, "ratio": 0.8795460281671181}}, {"step": 2600, "epoch": 5, "train_loss": 0.04901638254523277, "cos_r_negg": [0.4732121229171753, 0.5405805706977844, 0.5161677598953247], "cos_innovation_negg": [0.4732121229171753, 0.5405805706977844, 0.5161677598953247], "cos_apical_negg": [0.017585616558790207, 0.06275026500225067, 0.11934780329465866], "cos_Ac_negg": [0.5884664058685303, 0.63580322265625, 0.5768810510635376], "r_norm": [0.23610931634902954, 0.2359788715839386, 0.24158619344234467]}, {"step": 2700, "epoch": 5, "train_loss": 0.020244896411895752, "cos_r_negg": [0.5095292925834656, 0.5614259243011475, 0.5419886708259583], "cos_innovation_negg": [0.5095292925834656, 0.5614259243011475, 0.5419886708259583], "cos_apical_negg": [0.0031107051763683558, 0.054633576422929764, 0.10312239080667496], "cos_Ac_negg": [0.5985134840011597, 0.6457642316818237, 0.5994958877563477], "r_norm": [0.1804274171590805, 0.179721862077713, 0.18572017550468445]}, {"step": 2800, "epoch": 5, "train_loss": 0.03222280740737915, "cos_r_negg": [0.5060110092163086, 0.5528697371482849, 0.5404359698295593], "cos_innovation_negg": [0.5060110092163086, 0.5528697371482849, 0.5404359698295593], "cos_apical_negg": [0.0066470676101744175, 0.046586379408836365, 0.10817725956439972], "cos_Ac_negg": [0.6005290746688843, 0.6426056623458862, 0.5970289707183838], "r_norm": [0.2139023095369339, 0.21444171667099, 0.21976864337921143]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9702, "test_loss": 0.11684286804199219}, {"step": 2900, "epoch": 6, "train_loss": 0.013342317193746567, "cos_r_negg": [0.490291953086853, 0.5457422733306885, 0.5283840894699097], "cos_innovation_negg": [0.490291953086853, 0.5457422733306885, 0.5283840894699097], "cos_apical_negg": [0.01025344803929329, 0.04778910428285599, 0.10659874975681305], "cos_Ac_negg": [0.6051537990570068, 0.6483285427093506, 0.6009135246276855], "r_norm": [0.20289361476898193, 0.20486390590667725, 0.20754224061965942]}, {"step": 3000, "epoch": 6, "train_loss": 0.11073090136051178, "cos_r_negg": [0.4515626132488251, 0.5093556642532349, 0.5053510665893555], "cos_innovation_negg": [0.4515626132488251, 0.5093556642532349, 0.5053510665893555], "cos_apical_negg": [0.013737158849835396, 0.04446276277303696, 0.09778328239917755], "cos_Ac_negg": [0.617357611656189, 0.6529388427734375, 0.604745626449585], "r_norm": [0.17087693512439728, 0.17181196808815002, 0.17596760392189026], "ldr": {"dL_sdil": -0.004728503525257111, "dL_bp": -0.005845166742801666, "ratio": 0.8089595615181161}}, {"step": 3100, "epoch": 6, "train_loss": 0.0314796082675457, "cos_r_negg": [0.4605708420276642, 0.5072304010391235, 0.5078403949737549], "cos_innovation_negg": [0.4605708420276642, 0.5072304010391235, 0.5078403949737549], "cos_apical_negg": [0.008323969319462776, 0.04148902744054794, 0.09915271401405334], "cos_Ac_negg": [0.6148794293403625, 0.6553107500076294, 0.6123335361480713], "r_norm": [0.15782269835472107, 0.15876182913780212, 0.16421592235565186]}, {"step": 3200, "epoch": 6, "train_loss": 0.030566129833459854, "cos_r_negg": [0.45167309045791626, 0.49532195925712585, 0.5064538717269897], "cos_innovation_negg": [0.45167309045791626, 0.49532195925712585, 0.5064538717269897], "cos_apical_negg": [0.0010364686604589224, 0.04421323537826538, 0.10015524923801422], "cos_Ac_negg": [0.6173218488693237, 0.6547077298164368, 0.6201417446136475], "r_norm": [0.15362250804901123, 0.15379174053668976, 0.15875184535980225]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9732, "test_loss": 0.11364617080688477}, {"step": 3300, "epoch": 7, "train_loss": 0.012705174274742603, "cos_r_negg": [0.441884309053421, 0.48938363790512085, 0.49678999185562134], "cos_innovation_negg": [0.441884309053421, 0.48938363790512085, 0.49678999185562134], "cos_apical_negg": [0.001496356911957264, 0.04050331190228462, 0.09810030460357666], "cos_Ac_negg": [0.6149899959564209, 0.6517817378044128, 0.62260901927948], "r_norm": [0.144195556640625, 0.14456094801425934, 0.14890941977500916]}, {"step": 3400, "epoch": 7, "train_loss": 0.02708619460463524, "cos_r_negg": [0.4019877314567566, 0.4612293541431427, 0.4786125123500824], "cos_innovation_negg": [0.4019877314567566, 0.4612293541431427, 0.4786125123500824], "cos_apical_negg": [0.011159900575876236, 0.03742310032248497, 0.10522318631410599], "cos_Ac_negg": [0.608173131942749, 0.6513979434967041, 0.6207205653190613], "r_norm": [0.1775926649570465, 0.1788080930709839, 0.18419823050498962]}, {"step": 3500, "epoch": 7, "train_loss": 0.006117243319749832, "cos_r_negg": [0.3890681266784668, 0.4490724503993988, 0.47206851840019226], "cos_innovation_negg": [0.3890681266784668, 0.4490724503993988, 0.47206851840019226], "cos_apical_negg": [0.0006661583902314305, 0.040748827159404755, 0.10464087128639221], "cos_Ac_negg": [0.6045383214950562, 0.652137041091919, 0.6325887441635132], "r_norm": [0.17579853534698486, 0.17807260155677795, 0.18121156096458435], "ldr": {"dL_sdil": -0.006286561489105225, "dL_bp": -0.008487045764923096, "ratio": 0.7407243537070982}}, {"step": 3600, "epoch": 7, "train_loss": 0.0014728414826095104, "cos_r_negg": [0.3956383466720581, 0.44037681818008423, 0.46554142236709595], "cos_innovation_negg": [0.3956383466720581, 0.44037681818008423, 0.46554142236709595], "cos_apical_negg": [0.006933513097465038, 0.03878398984670639, 0.09817963093519211], "cos_Ac_negg": [0.6103466749191284, 0.6541887521743774, 0.6346606016159058], "r_norm": [0.17161166667938232, 0.17173579335212708, 0.1757110208272934]}, {"step": 3700, "epoch": 7, "train_loss": 0.013408890925347805, "cos_r_negg": [0.3940478563308716, 0.43435096740722656, 0.458285927772522], "cos_innovation_negg": [0.3940478563308716, 0.43435096740722656, 0.458285927772522], "cos_apical_negg": [0.008719246834516525, 0.040299706161022186, 0.0984993577003479], "cos_Ac_negg": [0.616531491279602, 0.6621330976486206, 0.6387621760368347], "r_norm": [0.15341749787330627, 0.15340277552604675, 0.15871842205524445]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9721, "test_loss": 0.11891266403198242}, {"step": 3800, "epoch": 8, "train_loss": 0.0017312548588961363, "cos_r_negg": [0.3702239692211151, 0.4182809591293335, 0.4442867636680603], "cos_innovation_negg": [0.3702239692211151, 0.4182809591293335, 0.4442867636680603], "cos_apical_negg": [0.00756513886153698, 0.04237908124923706, 0.09756618738174438], "cos_Ac_negg": [0.6189332008361816, 0.6612255573272705, 0.6436887979507446], "r_norm": [0.1580124944448471, 0.15794450044631958, 0.16437974572181702]}, {"step": 3900, "epoch": 8, "train_loss": 0.008029010146856308, "cos_r_negg": [0.3323814272880554, 0.38785019516944885, 0.42252957820892334], "cos_innovation_negg": [0.3323814272880554, 0.38785019516944885, 0.42252957820892334], "cos_apical_negg": [-0.0035301714669913054, 0.04037394002079964, 0.09997766464948654], "cos_Ac_negg": [0.6143934726715088, 0.6578063368797302, 0.6398673057556152], "r_norm": [0.16382494568824768, 0.16403745114803314, 0.16964119672775269]}, {"step": 4000, "epoch": 8, "train_loss": 0.005278876982629299, "cos_r_negg": [0.32559406757354736, 0.377626895904541, 0.40766674280166626], "cos_innovation_negg": [0.32559406757354736, 0.377626895904541, 0.40766674280166626], "cos_apical_negg": [0.0004269819473847747, 0.041124481707811356, 0.09857790917158127], "cos_Ac_negg": [0.6162174344062805, 0.6578550338745117, 0.6420688629150391], "r_norm": [0.1624622642993927, 0.16297931969165802, 0.16874642670154572], "ldr": {"dL_sdil": -0.0053291916847229, "dL_bp": -0.008842051029205322, "ratio": 0.6027098992214096}}, {"step": 4100, "epoch": 8, "train_loss": 0.00907666515558958, "cos_r_negg": [0.35214149951934814, 0.40032824873924255, 0.43064430356025696], "cos_innovation_negg": [0.35214149951934814, 0.40032824873924255, 0.43064430356025696], "cos_apical_negg": [0.01142967026680708, 0.038314394652843475, 0.0997210443019867], "cos_Ac_negg": [0.6214534640312195, 0.6580898761749268, 0.6477608680725098], "r_norm": [0.1661398708820343, 0.16598953306674957, 0.17114144563674927]}, {"step": 4200, "epoch": 8, "train_loss": 0.006652669981122017, "cos_r_negg": [0.3395025134086609, 0.3863486647605896, 0.4181085228919983], "cos_innovation_negg": [0.3395025134086609, 0.3863486647605896, 0.4181085228919983], "cos_apical_negg": [0.00505808275192976, 0.03965885192155838, 0.09239332377910614], "cos_Ac_negg": [0.6219109296798706, 0.6571305990219116, 0.6509281396865845], "r_norm": [0.17475536465644836, 0.17501097917556763, 0.17999820411205292]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9734, "test_loss": 0.12600592975616456}, {"step": 4300, "epoch": 9, "train_loss": 0.000920439837500453, "cos_r_negg": [0.31656304001808167, 0.36605653166770935, 0.39582252502441406], "cos_innovation_negg": [0.31656304001808167, 0.36605653166770935, 0.39582252502441406], "cos_apical_negg": [0.011951670050621033, 0.03702542558312416, 0.09015651047229767], "cos_Ac_negg": [0.6237709522247314, 0.657106876373291, 0.6498638391494751], "r_norm": [0.1676049679517746, 0.1691950410604477, 0.17278146743774414]}, {"step": 4400, "epoch": 9, "train_loss": 0.0015409726183861494, "cos_r_negg": [0.32093679904937744, 0.3758125305175781, 0.40481477975845337], "cos_innovation_negg": [0.32093679904937744, 0.3758125305175781, 0.40481477975845337], "cos_apical_negg": [0.0014109625481069088, 0.03958212584257126, 0.09350169450044632], "cos_Ac_negg": [0.6212763786315918, 0.6606855392456055, 0.6536861062049866], "r_norm": [0.1716303825378418, 0.1719321310520172, 0.17656877636909485]}, {"step": 4500, "epoch": 9, "train_loss": 0.070710189640522, "cos_r_negg": [0.2866787910461426, 0.3375962972640991, 0.3757855296134949], "cos_innovation_negg": [0.2866787910461426, 0.3375962972640991, 0.3757855296134949], "cos_apical_negg": [0.004375724121928215, 0.0351162813603878, 0.095660500228405], "cos_Ac_negg": [0.604589581489563, 0.6557512283325195, 0.6560600399971008], "r_norm": [0.17863571643829346, 0.1787968873977661, 0.18372419476509094], "ldr": {"dL_sdil": -0.00881260633468628, "dL_bp": -0.014440789818763733, "ratio": 0.6102579183886163}}, {"step": 4600, "epoch": 9, "train_loss": 0.00025532543077133596, "cos_r_negg": [0.30047187209129333, 0.35224369168281555, 0.38576561212539673], "cos_innovation_negg": [0.30047187209129333, 0.35224369168281555, 0.38576561212539673], "cos_apical_negg": [-0.004670341964811087, 0.03179685026407242, 0.0876118615269661], "cos_Ac_negg": [0.6112259030342102, 0.6568924188613892, 0.654515266418457], "r_norm": [0.14777541160583496, 0.14828453958034515, 0.15262819826602936]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9746, "test_loss": 0.12607035331726074}, {"step": 4700, "epoch": 10, "train_loss": 0.0018456950783729553, "cos_r_negg": [0.2685810625553131, 0.321600079536438, 0.35757583379745483], "cos_innovation_negg": [0.2685810625553131, 0.321600079536438, 0.35757583379745483], "cos_apical_negg": [-0.0023618494160473347, 0.034356892108917236, 0.09040367603302002], "cos_Ac_negg": [0.6116015315055847, 0.6569674015045166, 0.6582921743392944], "r_norm": [0.1558665633201599, 0.15650692582130432, 0.16111719608306885]}, {"step": 4800, "epoch": 10, "train_loss": 0.0019524649251252413, "cos_r_negg": [0.2605072557926178, 0.3158532381057739, 0.35141417384147644], "cos_innovation_negg": [0.2605072557926178, 0.3158532381057739, 0.35141417384147644], "cos_apical_negg": [-0.006188189145177603, 0.03293583169579506, 0.08731848001480103], "cos_Ac_negg": [0.6076411008834839, 0.6547963619232178, 0.6580339074134827], "r_norm": [0.15249508619308472, 0.15310779213905334, 0.15842777490615845]}, {"step": 4900, "epoch": 10, "train_loss": 0.002264401875436306, "cos_r_negg": [0.2611931264400482, 0.3164622187614441, 0.34751421213150024], "cos_innovation_negg": [0.2611931264400482, 0.3164622187614441, 0.34751421213150024], "cos_apical_negg": [-0.008969413116574287, 0.03165031224489212, 0.08151207119226456], "cos_Ac_negg": [0.6055455207824707, 0.6518632173538208, 0.6556879281997681], "r_norm": [0.14897800981998444, 0.1497153788805008, 0.1550256609916687]}, {"step": 5000, "epoch": 10, "train_loss": 0.0010343781905248761, "cos_r_negg": [0.26585742831230164, 0.31862691044807434, 0.3502918481826782], "cos_innovation_negg": [0.26585742831230164, 0.31862691044807434, 0.3502918481826782], "cos_apical_negg": [-0.0055550807155668736, 0.031946003437042236, 0.08321933448314667], "cos_Ac_negg": [0.6045143604278564, 0.6546858549118042, 0.6582672595977783], "r_norm": [0.157735675573349, 0.15817727148532867, 0.163752019405365], "ldr": {"dL_sdil": -0.00658772885799408, "dL_bp": -0.01139974594116211, "ratio": 0.577883831095495}}, {"step": 5100, "epoch": 10, "train_loss": 0.0059285457246005535, "cos_r_negg": [0.2620428800582886, 0.3137778043746948, 0.3459997773170471], "cos_innovation_negg": [0.2620428800582886, 0.3137778043746948, 0.3459997773170471], "cos_apical_negg": [-0.0046820444986224174, 0.03465544432401657, 0.08417428284883499], "cos_Ac_negg": [0.6089699268341064, 0.6560293436050415, 0.6612058877944946], "r_norm": [0.15331438183784485, 0.153997004032135, 0.15881389379501343]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9749, "test_loss": 0.1303539514541626}, {"step": 5200, "epoch": 11, "train_loss": 0.0024754097685217857, "cos_r_negg": [0.2521519362926483, 0.30828967690467834, 0.3383101522922516], "cos_innovation_negg": [0.2521519362926483, 0.30828967690467834, 0.3383101522922516], "cos_apical_negg": [-0.006769408471882343, 0.03487198427319527, 0.08185990154743195], "cos_Ac_negg": [0.6064837574958801, 0.6537446975708008, 0.6618554592132568], "r_norm": [0.15417413413524628, 0.15528428554534912, 0.15972323715686798]}, {"step": 5300, "epoch": 11, "train_loss": 0.001265630591660738, "cos_r_negg": [0.24525953829288483, 0.30236753821372986, 0.3332999646663666], "cos_innovation_negg": [0.24525953829288483, 0.30236753821372986, 0.3332999646663666], "cos_apical_negg": [-0.007318840362131596, 0.03309658542275429, 0.08009163290262222], "cos_Ac_negg": [0.6042869687080383, 0.652653694152832, 0.6629073619842529], "r_norm": [0.1384962499141693, 0.13895532488822937, 0.1438809335231781]}, {"step": 5400, "epoch": 11, "train_loss": 0.0010879106121137738, "cos_r_negg": [0.24348558485507965, 0.2980152368545532, 0.3290289342403412], "cos_innovation_negg": [0.24348558485507965, 0.2980152368545532, 0.3290289342403412], "cos_apical_negg": [-0.007456302642822266, 0.03289063274860382, 0.08186434209346771], "cos_Ac_negg": [0.6037515997886658, 0.6524885892868042, 0.6619837284088135], "r_norm": [0.14330285787582397, 0.14409542083740234, 0.14913520216941833]}, {"step": 5500, "epoch": 11, "train_loss": 0.0010222081327810884, "cos_r_negg": [0.24391423165798187, 0.29650700092315674, 0.32684457302093506], "cos_innovation_negg": [0.24391423165798187, 0.29650700092315674, 0.32684457302093506], "cos_apical_negg": [-0.004356726072728634, 0.03291890025138855, 0.08121198415756226], "cos_Ac_negg": [0.6020787358283997, 0.6529993414878845, 0.6636874675750732], "r_norm": [0.15417656302452087, 0.15446804463863373, 0.1599593162536621], "ldr": {"dL_sdil": -0.006770998239517212, "dL_bp": -0.012395210564136505, "ratio": 0.5462592349264301}}, {"step": 5600, "epoch": 11, "train_loss": 0.0008564281743019819, "cos_r_negg": [0.23328517377376556, 0.28719162940979004, 0.3161482810974121], "cos_innovation_negg": [0.23328517377376556, 0.28719162940979004, 0.3161482810974121], "cos_apical_negg": [-0.006658114958554506, 0.03184705600142479, 0.07988293468952179], "cos_Ac_negg": [0.5983502864837646, 0.6513689756393433, 0.6641026139259338], "r_norm": [0.15335647761821747, 0.154288649559021, 0.15924350917339325]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9751, "test_loss": 0.13479326438903808}, {"step": 5700, "epoch": 12, "train_loss": 0.0010083125671371818, "cos_r_negg": [0.2259504497051239, 0.2766932547092438, 0.30644237995147705], "cos_innovation_negg": [0.2259504497051239, 0.2766932547092438, 0.30644237995147705], "cos_apical_negg": [-0.006252835504710674, 0.03170429542660713, 0.07870615273714066], "cos_Ac_negg": [0.5965291261672974, 0.6469354629516602, 0.6624070405960083], "r_norm": [0.14757829904556274, 0.14831095933914185, 0.15284687280654907]}, {"step": 5800, "epoch": 12, "train_loss": 0.0007182832923717797, "cos_r_negg": [0.22713306546211243, 0.2781926095485687, 0.30721211433410645], "cos_innovation_negg": [0.22713306546211243, 0.2781926095485687, 0.30721211433410645], "cos_apical_negg": [-0.005868683103471994, 0.03285183012485504, 0.07951490581035614], "cos_Ac_negg": [0.5939555168151855, 0.6486457586288452, 0.6644296646118164], "r_norm": [0.15043538808822632, 0.15142518281936646, 0.15595465898513794]}, {"step": 5900, "epoch": 12, "train_loss": 0.0011099032126367092, "cos_r_negg": [0.22418621182441711, 0.2767918109893799, 0.3060784339904785], "cos_innovation_negg": [0.22418621182441711, 0.2767918109893799, 0.3060784339904785], "cos_apical_negg": [-0.005779815837740898, 0.03284506872296333, 0.07832050323486328], "cos_Ac_negg": [0.5937525033950806, 0.6487970352172852, 0.6661155819892883], "r_norm": [0.15428504347801208, 0.15492992103099823, 0.15943790972232819]}, {"step": 6000, "epoch": 12, "train_loss": 0.0006464760517701507, "cos_r_negg": [0.22026482224464417, 0.27408331632614136, 0.3008838891983032], "cos_innovation_negg": [0.22026482224464417, 0.27408331632614136, 0.3008838891983032], "cos_apical_negg": [-0.006945154629647732, 0.032189808785915375, 0.07708658277988434], "cos_Ac_negg": [0.5939316749572754, 0.6466071009635925, 0.664664626121521], "r_norm": [0.15557214617729187, 0.15675458312034607, 0.16110199689865112], "ldr": {"dL_sdil": -0.007881097495555878, "dL_bp": -0.015217460691928864, "ratio": 0.5178983310754275}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9753, "test_loss": 0.13703663864135743}, {"step": 6100, "epoch": 13, "train_loss": 0.00047011717106215656, "cos_r_negg": [0.2155582159757614, 0.2691652774810791, 0.29353076219558716], "cos_innovation_negg": [0.2155582159757614, 0.2691652774810791, 0.29353076219558716], "cos_apical_negg": [-0.00823905598372221, 0.032206322997808456, 0.075456902384758], "cos_Ac_negg": [0.5926049947738647, 0.6450304985046387, 0.6638611555099487], "r_norm": [0.15490058064460754, 0.15590539574623108, 0.16053014993667603]}, {"step": 6200, "epoch": 13, "train_loss": 0.0006191445281729102, "cos_r_negg": [0.21145445108413696, 0.2632468044757843, 0.2880074977874756], "cos_innovation_negg": [0.21145445108413696, 0.2632468044757843, 0.2880074977874756], "cos_apical_negg": [-0.00477974209934473, 0.032112378627061844, 0.07395368069410324], "cos_Ac_negg": [0.5930963754653931, 0.6443892121315002, 0.665156364440918], "r_norm": [0.14573867619037628, 0.14631709456443787, 0.15107320249080658]}, {"step": 6300, "epoch": 13, "train_loss": 0.0006084823980927467, "cos_r_negg": [0.20774081349372864, 0.2611802816390991, 0.2843462824821472], "cos_innovation_negg": [0.20774081349372864, 0.2611802816390991, 0.2843462824821472], "cos_apical_negg": [-0.005467954091727734, 0.03359202295541763, 0.07382342964410782], "cos_Ac_negg": [0.5903453826904297, 0.6435489654541016, 0.6650875806808472], "r_norm": [0.1558074653148651, 0.1568615436553955, 0.16144248843193054]}, {"step": 6400, "epoch": 13, "train_loss": 0.00029535041539929807, "cos_r_negg": [0.21015006303787231, 0.26090872287750244, 0.28448501229286194], "cos_innovation_negg": [0.21015006303787231, 0.26090872287750244, 0.28448501229286194], "cos_apical_negg": [-0.006234245374798775, 0.03328931704163551, 0.07406023144721985], "cos_Ac_negg": [0.5899637937545776, 0.6449921727180481, 0.6659203171730042], "r_norm": [0.1541547328233719, 0.15523368120193481, 0.15950702130794525]}, {"step": 6500, "epoch": 13, "train_loss": 0.0001267417101189494, "cos_r_negg": [0.20930953323841095, 0.26244157552719116, 0.28575050830841064], "cos_innovation_negg": [0.20930953323841095, 0.26244157552719116, 0.28575050830841064], "cos_apical_negg": [-0.00786194670945406, 0.03217038884758949, 0.07371644675731659], "cos_Ac_negg": [0.5902331471443176, 0.6453607082366943, 0.6665254831314087], "r_norm": [0.1492280513048172, 0.15031403303146362, 0.15442019701004028], "ldr": {"dL_sdil": -0.007210694253444672, "dL_bp": -0.0137811079621315, "ratio": 0.5232303725693624}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9747, "test_loss": 0.13892102909088136}, {"step": 6600, "epoch": 14, "train_loss": 0.00033589144004508853, "cos_r_negg": [0.2012377679347992, 0.2551075518131256, 0.2807343602180481], "cos_innovation_negg": [0.2012377679347992, 0.2551075518131256, 0.2807343602180481], "cos_apical_negg": [-0.008127100765705109, 0.032083939760923386, 0.07228269428014755], "cos_Ac_negg": [0.5913076400756836, 0.6444406509399414, 0.6687202453613281], "r_norm": [0.14491665363311768, 0.1456868052482605, 0.14983847737312317]}, {"step": 6700, "epoch": 14, "train_loss": 0.0003879019641317427, "cos_r_negg": [0.20080190896987915, 0.25391122698783875, 0.2786269187927246], "cos_innovation_negg": [0.20080190896987915, 0.25391122698783875, 0.2786269187927246], "cos_apical_negg": [-0.005730796605348587, 0.03303510323166847, 0.07268358767032623], "cos_Ac_negg": [0.5866837501525879, 0.6435134410858154, 0.6694850921630859], "r_norm": [0.15811243653297424, 0.15914101898670197, 0.16268214583396912]}, {"step": 6800, "epoch": 14, "train_loss": 0.0007882924983277917, "cos_r_negg": [0.19695380330085754, 0.24855253100395203, 0.273712158203125], "cos_innovation_negg": [0.19695380330085754, 0.24855253100395203, 0.273712158203125], "cos_apical_negg": [-0.007303585764020681, 0.03210416063666344, 0.07287061214447021], "cos_Ac_negg": [0.5861085057258606, 0.6424258947372437, 0.6689814329147339], "r_norm": [0.15456651151180267, 0.1555781066417694, 0.1594783216714859]}, {"step": 6900, "epoch": 14, "train_loss": 0.0005307328538037837, "cos_r_negg": [0.1942259669303894, 0.24703101813793182, 0.27093228697776794], "cos_innovation_negg": [0.1942259669303894, 0.24703101813793182, 0.27093228697776794], "cos_apical_negg": [-0.009035391733050346, 0.032507091760635376, 0.07144492864608765], "cos_Ac_negg": [0.5867035388946533, 0.6425132155418396, 0.6696616411209106], "r_norm": [0.14989393949508667, 0.15100210905075073, 0.1547473669052124]}, {"step": 7000, "epoch": 14, "train_loss": 0.0004972174065187573, "cos_r_negg": [0.1940956711769104, 0.2456737756729126, 0.2701438367366791], "cos_innovation_negg": [0.1940956711769104, 0.2456737756729126, 0.2701438367366791], "cos_apical_negg": [-0.009482834488153458, 0.030541446059942245, 0.07118197530508041], "cos_Ac_negg": [0.5875210762023926, 0.6431896090507507, 0.6700403094291687], "r_norm": [0.141020730137825, 0.14182855188846588, 0.14628368616104126], "ldr": {"dL_sdil": -0.0060822367668151855, "dL_bp": -0.011987894773483276, "ratio": 0.5073648777989643}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9749, "test_loss": 0.14240283432006837}], "final": {"test_acc": 0.9749, "test_loss": 0.14240283432006837, "wall_s": 23.527334213256836, "cos_r_negg": [0.1927020400762558, 0.2439565658569336, 0.26908159255981445], "cos_innovation_negg": [0.1927020400762558, 0.2439565658569336, 0.26908159255981445], "cos_apical_negg": [-0.008781068958342075, 0.030784491449594498, 0.07170166820287704], "cos_Ac_negg": [0.5875163078308105, 0.6434217691421509, 0.6704922318458557]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_residual_s3.json b/results/nuis_ctrl_v1_rho0p5_residual_s3.json new file mode 100644 index 0000000..99791e8 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_residual_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_residual_s3", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.331878900527954, "cos_r_negg": [-0.014011913910508156, 0.013712333515286446, 0.00608993461355567], "cos_innovation_negg": [-0.014011913910508156, 0.013712333515286446, 0.00608993461355567], "cos_apical_negg": [-0.018168717622756958, 0.005241477862000465, -0.002439321018755436], "cos_Ac_negg": [-0.014013777486979961, 0.01371200755238533, 0.006088861729949713], "r_norm": [4.854389667510986, 4.749159336090088, 4.848528861999512], "ldr": {"dL_sdil": -0.005772829055786133, "dL_bp": -0.23062801361083984, "ratio": 0.025030910015672102}}, {"step": 100, "epoch": 0, "train_loss": 0.18872550129890442, "cos_r_negg": [0.42080679535865784, 0.3573850691318512, 0.31352412700653076], "cos_innovation_negg": [0.42080679535865784, 0.3573850691318512, 0.31352412700653076], "cos_apical_negg": [0.07740601152181625, 0.08406104147434235, 0.12225973606109619], "cos_Ac_negg": [0.4255574345588684, 0.35864710807800293, 0.312264084815979], "r_norm": [0.8661477565765381, 0.8538812398910522, 0.8630181550979614]}, {"step": 200, "epoch": 0, "train_loss": 0.23271331191062927, "cos_r_negg": [0.47490373253822327, 0.4291550815105438, 0.35083863139152527], "cos_innovation_negg": [0.47490373253822327, 0.4291550815105438, 0.35083863139152527], "cos_apical_negg": [0.06322602927684784, 0.08257284760475159, 0.11042001843452454], "cos_Ac_negg": [0.479240357875824, 0.43211889266967773, 0.3495291471481323], "r_norm": [0.7295346856117249, 0.7159911394119263, 0.7286012172698975]}, {"step": 300, "epoch": 0, "train_loss": 0.15973104536533356, "cos_r_negg": [0.4891892969608307, 0.4489712119102478, 0.387234091758728], "cos_innovation_negg": [0.4891892969608307, 0.4489712119102478, 0.387234091758728], "cos_apical_negg": [0.05702277645468712, 0.0757027268409729, 0.11901925504207611], "cos_Ac_negg": [0.4903077483177185, 0.45112019777297974, 0.3880354166030884], "r_norm": [0.6274819374084473, 0.6219789981842041, 0.626327395439148]}, {"step": 400, "epoch": 0, "train_loss": 0.15213412046432495, "cos_r_negg": [0.5123450756072998, 0.4869316816329956, 0.4288489818572998], "cos_innovation_negg": [0.5123450756072998, 0.4869316816329956, 0.4288489818572998], "cos_apical_negg": [0.04249223321676254, 0.0700196623802185, 0.09839313477277756], "cos_Ac_negg": [0.5162568092346191, 0.4874834716320038, 0.42815375328063965], "r_norm": [0.5795223712921143, 0.5721928477287292, 0.5810239315032959]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9473, "test_loss": 0.1803105567932129}, {"step": 500, "epoch": 1, "train_loss": 0.2707977890968323, "cos_r_negg": [0.4892926812171936, 0.47885721921920776, 0.4356996417045593], "cos_innovation_negg": [0.4892926812171936, 0.47885721921920776, 0.4356996417045593], "cos_apical_negg": [0.04015825316309929, 0.06950805336236954, 0.1100805476307869], "cos_Ac_negg": [0.49992287158966064, 0.48546868562698364, 0.43970683217048645], "r_norm": [0.5381065607070923, 0.5282456874847412, 0.5357717871665955], "ldr": {"dL_sdil": -0.018530339002609253, "dL_bp": -0.01635301113128662, "ratio": 1.1331453793948054}}, {"step": 600, "epoch": 1, "train_loss": 0.11664538830518723, "cos_r_negg": [0.49489182233810425, 0.5053346157073975, 0.443124383687973], "cos_innovation_negg": [0.49489182233810425, 0.5053346157073975, 0.443124383687973], "cos_apical_negg": [0.022730866447091103, 0.05235520377755165, 0.09304158389568329], "cos_Ac_negg": [0.504249632358551, 0.5136399865150452, 0.44650179147720337], "r_norm": [0.4068072438240051, 0.3990352749824524, 0.40562427043914795]}, {"step": 700, "epoch": 1, "train_loss": 0.17657727003097534, "cos_r_negg": [0.5248271226882935, 0.5245544910430908, 0.45468053221702576], "cos_innovation_negg": [0.5248271226882935, 0.5245544910430908, 0.45468053221702576], "cos_apical_negg": [0.03518783301115036, 0.06547833234071732, 0.10126242786645889], "cos_Ac_negg": [0.5350420475006104, 0.532680869102478, 0.4575701951980591], "r_norm": [0.4497723877429962, 0.44423526525497437, 0.4508831799030304]}, {"step": 800, "epoch": 1, "train_loss": 0.13849526643753052, "cos_r_negg": [0.5045038461685181, 0.5329877734184265, 0.47357553243637085], "cos_innovation_negg": [0.5045038461685181, 0.5329877734184265, 0.47357553243637085], "cos_apical_negg": [0.025899339467287064, 0.04956956207752228, 0.10712993144989014], "cos_Ac_negg": [0.5124516487121582, 0.5372099280357361, 0.4743672013282776], "r_norm": [0.3278277516365051, 0.32419031858444214, 0.327649861574173]}, {"step": 900, "epoch": 1, "train_loss": 0.1713041067123413, "cos_r_negg": [0.5401734709739685, 0.549518883228302, 0.47928205132484436], "cos_innovation_negg": [0.5401734709739685, 0.549518883228302, 0.47928205132484436], "cos_apical_negg": [0.0506848581135273, 0.059160977602005005, 0.09672336280345917], "cos_Ac_negg": [0.541490375995636, 0.5504716634750366, 0.4794657528400421], "r_norm": [0.4373318552970886, 0.430068701505661, 0.43439656496047974]}, {"epoch_end": 1, "step": 938, "test_acc": 0.9561, "test_loss": 0.15255565452575684}, {"step": 1000, "epoch": 2, "train_loss": 0.02094106562435627, "cos_r_negg": [0.5435171127319336, 0.5617809891700745, 0.48627057671546936], "cos_innovation_negg": [0.5435171127319336, 0.5617809891700745, 0.48627057671546936], "cos_apical_negg": [0.03208060935139656, 0.06277690082788467, 0.09445619583129883], "cos_Ac_negg": [0.546578586101532, 0.5651637315750122, 0.4887245297431946], "r_norm": [0.31476879119873047, 0.31028246879577637, 0.3133445382118225], "ldr": {"dL_sdil": -0.006925120949745178, "dL_bp": -0.00616048276424408, "ratio": 1.1241198481942223}}, {"step": 1100, "epoch": 2, "train_loss": 0.09473104029893875, "cos_r_negg": [0.5680637955665588, 0.5742523670196533, 0.5016100406646729], "cos_innovation_negg": [0.5680637955665588, 0.5742523670196533, 0.5016100406646729], "cos_apical_negg": [0.037956446409225464, 0.05853698402643204, 0.09649095684289932], "cos_Ac_negg": [0.5749331712722778, 0.5808276534080505, 0.5028795003890991], "r_norm": [0.31742265820503235, 0.3098963499069214, 0.31269025802612305]}, {"step": 1200, "epoch": 2, "train_loss": 0.1336817443370819, "cos_r_negg": [0.5417118072509766, 0.5653090476989746, 0.5028981566429138], "cos_innovation_negg": [0.5417118072509766, 0.5653090476989746, 0.5028981566429138], "cos_apical_negg": [0.030298395082354546, 0.0590420663356781, 0.09857246279716492], "cos_Ac_negg": [0.5533450841903687, 0.5780097246170044, 0.5068104267120361], "r_norm": [0.3133755326271057, 0.30838990211486816, 0.3106110692024231]}, {"step": 1300, "epoch": 2, "train_loss": 0.115728460252285, "cos_r_negg": [0.5507748126983643, 0.5746300220489502, 0.5097249746322632], "cos_innovation_negg": [0.5507748126983643, 0.5746300220489502, 0.5097249746322632], "cos_apical_negg": [0.03266730159521103, 0.056138359010219574, 0.08230951428413391], "cos_Ac_negg": [0.5631198883056641, 0.5865797996520996, 0.5113161206245422], "r_norm": [0.30199965834617615, 0.29994842410087585, 0.3007659912109375]}, {"step": 1400, "epoch": 2, "train_loss": 0.16870275139808655, "cos_r_negg": [0.5444096326828003, 0.5728497505187988, 0.5018571019172668], "cos_innovation_negg": [0.5444096326828003, 0.5728497505187988, 0.5018571019172668], "cos_apical_negg": [0.025644447654485703, 0.05886274576187134, 0.07878376543521881], "cos_Ac_negg": [0.5628910064697266, 0.588857889175415, 0.5121197700500488], "r_norm": [0.27241218090057373, 0.2691349387168884, 0.27151989936828613]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9657, "test_loss": 0.11589976081848144}, {"step": 1500, "epoch": 3, "train_loss": 0.06594803929328918, "cos_r_negg": [0.5402629375457764, 0.5796506404876709, 0.5273761749267578], "cos_innovation_negg": [0.5402629375457764, 0.5796506404876709, 0.5273761749267578], "cos_apical_negg": [0.039710983633995056, 0.05154367908835411, 0.0969703197479248], "cos_Ac_negg": [0.5638954043388367, 0.601933479309082, 0.535617470741272], "r_norm": [0.27322760224342346, 0.26843664050102234, 0.26908040046691895], "ldr": {"dL_sdil": -0.0067085325717926025, "dL_bp": -0.006532587110996246, "ratio": 1.0269335039559118}}, {"step": 1600, "epoch": 3, "train_loss": 0.0769442692399025, "cos_r_negg": [0.5356593132019043, 0.5775347352027893, 0.5283666253089905], "cos_innovation_negg": [0.5356593132019043, 0.5775347352027893, 0.5283666253089905], "cos_apical_negg": [0.03112872503697872, 0.05700736492872238, 0.09994073212146759], "cos_Ac_negg": [0.5571012496948242, 0.6037145853042603, 0.5409438014030457], "r_norm": [0.26720550656318665, 0.2618980407714844, 0.263634592294693]}, {"step": 1700, "epoch": 3, "train_loss": 0.04415497928857803, "cos_r_negg": [0.5329152345657349, 0.5690578818321228, 0.519217848777771], "cos_innovation_negg": [0.5329152345657349, 0.5690578818321228, 0.519217848777771], "cos_apical_negg": [0.03167639672756195, 0.051312316209077835, 0.09680252522230148], "cos_Ac_negg": [0.570207953453064, 0.6094943284988403, 0.5474331378936768], "r_norm": [0.27063578367233276, 0.2675698697566986, 0.2672215402126312]}, {"step": 1800, "epoch": 3, "train_loss": 0.020682916045188904, "cos_r_negg": [0.543427586555481, 0.5755721926689148, 0.5407392382621765], "cos_innovation_negg": [0.543427586555481, 0.5755721926689148, 0.5407392382621765], "cos_apical_negg": [0.020985178649425507, 0.051615044474601746, 0.092034250497818], "cos_Ac_negg": [0.5796080827713013, 0.6094875335693359, 0.5618948936462402], "r_norm": [0.228158101439476, 0.22647729516029358, 0.22639253735542297]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9647, "test_loss": 0.12295613803863525}, {"step": 1900, "epoch": 4, "train_loss": 0.018825624138116837, "cos_r_negg": [0.528862476348877, 0.5797684192657471, 0.5388694405555725], "cos_innovation_negg": [0.528862476348877, 0.5797684192657471, 0.5388694405555725], "cos_apical_negg": [0.029006794095039368, 0.05604682117700577, 0.10635900497436523], "cos_Ac_negg": [0.5644512176513672, 0.6147205829620361, 0.5558013916015625], "r_norm": [0.28968456387519836, 0.28626900911331177, 0.2880341708660126]}, {"step": 2000, "epoch": 4, "train_loss": 0.12200286239385605, "cos_r_negg": [0.49291181564331055, 0.5381568670272827, 0.5108394026756287], "cos_innovation_negg": [0.49291181564331055, 0.5381568670272827, 0.5108394026756287], "cos_apical_negg": [0.036747612059116364, 0.0550658255815506, 0.10144195705652237], "cos_Ac_negg": [0.5494893789291382, 0.6038867235183716, 0.5505704879760742], "r_norm": [0.24793246388435364, 0.24428686499595642, 0.24390989542007446], "ldr": {"dL_sdil": -0.007415145635604858, "dL_bp": -0.008610181510448456, "ratio": 0.8612066570962039}}, {"step": 2100, "epoch": 4, "train_loss": 0.028551729395985603, "cos_r_negg": [0.5037931799888611, 0.5556021332740784, 0.5132607221603394], "cos_innovation_negg": [0.5037931799888611, 0.5556021332740784, 0.5132607221603394], "cos_apical_negg": [0.013839269056916237, 0.058557067066431046, 0.10582729429006577], "cos_Ac_negg": [0.5713272094726562, 0.6184545755386353, 0.5559008121490479], "r_norm": [0.24037976562976837, 0.23496440052986145, 0.23672522604465485]}, {"step": 2200, "epoch": 4, "train_loss": 0.009481847286224365, "cos_r_negg": [0.5175397992134094, 0.5628323554992676, 0.5348480939865112], "cos_innovation_negg": [0.5175397992134094, 0.5628323554992676, 0.5348480939865112], "cos_apical_negg": [0.0159706249833107, 0.04661000892519951, 0.09774476289749146], "cos_Ac_negg": [0.5720933675765991, 0.6177525520324707, 0.5666742324829102], "r_norm": [0.22913230955600739, 0.22418493032455444, 0.2254614233970642]}, {"step": 2300, "epoch": 4, "train_loss": 0.1396801620721817, "cos_r_negg": [0.4930618405342102, 0.5409141182899475, 0.5277037620544434], "cos_innovation_negg": [0.4930618405342102, 0.5409141182899475, 0.5277037620544434], "cos_apical_negg": [0.008156484924256802, 0.0498521588742733, 0.09864520281553268], "cos_Ac_negg": [0.571165144443512, 0.6128248572349548, 0.5722683668136597], "r_norm": [0.20367828011512756, 0.19950716197490692, 0.2002502977848053]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9707, "test_loss": 0.10883730421066284}, {"step": 2400, "epoch": 5, "train_loss": 0.06521674990653992, "cos_r_negg": [0.4649459421634674, 0.5170556306838989, 0.5078390836715698], "cos_innovation_negg": [0.4649459421634674, 0.5170556306838989, 0.5078390836715698], "cos_apical_negg": [0.0044569773599505424, 0.049284402281045914, 0.10828302800655365], "cos_Ac_negg": [0.579290509223938, 0.6281781792640686, 0.5828486680984497], "r_norm": [0.18944184482097626, 0.18613117933273315, 0.18644651770591736]}, {"step": 2500, "epoch": 5, "train_loss": 0.012442920356988907, "cos_r_negg": [0.44050684571266174, 0.490290105342865, 0.4901142716407776], "cos_innovation_negg": [0.44050684571266174, 0.490290105342865, 0.4901142716407776], "cos_apical_negg": [0.014281416311860085, 0.05564771592617035, 0.10965365171432495], "cos_Ac_negg": [0.5732725858688354, 0.6313501596450806, 0.582327127456665], "r_norm": [0.24482318758964539, 0.2420414686203003, 0.2414052039384842], "ldr": {"dL_sdil": -0.008126087486743927, "dL_bp": -0.008378878235816956, "ratio": 0.9698300008714256}}, {"step": 2600, "epoch": 5, "train_loss": 0.023398997262120247, "cos_r_negg": [0.4568743109703064, 0.5034669637680054, 0.5188430547714233], "cos_innovation_negg": [0.4568743109703064, 0.5034669637680054, 0.5188430547714233], "cos_apical_negg": [0.01007128320634365, 0.046921633183956146, 0.10567986965179443], "cos_Ac_negg": [0.586541473865509, 0.6357093453407288, 0.6017983555793762], "r_norm": [0.21272389590740204, 0.20777344703674316, 0.20944646000862122]}, {"step": 2700, "epoch": 5, "train_loss": 0.04132649302482605, "cos_r_negg": [0.4660347104072571, 0.5183552503585815, 0.5186158418655396], "cos_innovation_negg": [0.4660347104072571, 0.5183552503585815, 0.5186158418655396], "cos_apical_negg": [0.009593063965439796, 0.04411444813013077, 0.10364066809415817], "cos_Ac_negg": [0.5876105427742004, 0.6435339450836182, 0.6019076108932495], "r_norm": [0.17805656790733337, 0.17355357110500336, 0.17494475841522217]}, {"step": 2800, "epoch": 5, "train_loss": 0.05498193949460983, "cos_r_negg": [0.43562740087509155, 0.4989575743675232, 0.5196200609207153], "cos_innovation_negg": [0.43562740087509155, 0.4989575743675232, 0.5196200609207153], "cos_apical_negg": [0.005082856398075819, 0.04511084035038948, 0.1107931137084961], "cos_Ac_negg": [0.5642528533935547, 0.642410159111023, 0.6124656200408936], "r_norm": [0.20564180612564087, 0.20171013474464417, 0.20210757851600647]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9693, "test_loss": 0.11708884296417237}, {"step": 2900, "epoch": 6, "train_loss": 0.005203034728765488, "cos_r_negg": [0.4495949149131775, 0.5044668912887573, 0.5137101411819458], "cos_innovation_negg": [0.4495949149131775, 0.5044668912887573, 0.5137101411819458], "cos_apical_negg": [0.0029116824734956026, 0.04600415378808975, 0.10258268564939499], "cos_Ac_negg": [0.5837121605873108, 0.6465179920196533, 0.6148958206176758], "r_norm": [0.17525489628314972, 0.17214038968086243, 0.173087939620018]}, {"step": 3000, "epoch": 6, "train_loss": 0.08553176373243332, "cos_r_negg": [0.4206857979297638, 0.47899866104125977, 0.4925500452518463], "cos_innovation_negg": [0.4206857979297638, 0.47899866104125977, 0.4925500452518463], "cos_apical_negg": [-0.009011526592075825, 0.03955311328172684, 0.10614622384309769], "cos_Ac_negg": [0.5796300172805786, 0.6412211060523987, 0.6063554883003235], "r_norm": [0.17875617742538452, 0.1751418262720108, 0.17465780675411224], "ldr": {"dL_sdil": -0.004441358149051666, "dL_bp": -0.0056234002113342285, "ratio": 0.7897994064338333}}, {"step": 3100, "epoch": 6, "train_loss": 0.042554669082164764, "cos_r_negg": [0.40734750032424927, 0.4535464346408844, 0.4807383120059967], "cos_innovation_negg": [0.40734750032424927, 0.4535464346408844, 0.4807383120059967], "cos_apical_negg": [-0.0014351322315633297, 0.03926960751414299, 0.10535650700330734], "cos_Ac_negg": [0.589215099811554, 0.6471856832504272, 0.6189682483673096], "r_norm": [0.186112180352211, 0.18383631110191345, 0.18407349288463593]}, {"step": 3200, "epoch": 6, "train_loss": 0.047677651047706604, "cos_r_negg": [0.40638357400894165, 0.4533464312553406, 0.4813411831855774], "cos_innovation_negg": [0.40638357400894165, 0.4533464312553406, 0.4813411831855774], "cos_apical_negg": [0.00492829829454422, 0.0322842076420784, 0.10239633172750473], "cos_Ac_negg": [0.5803347229957581, 0.6529821157455444, 0.6237295866012573], "r_norm": [0.1556628942489624, 0.15382838249206543, 0.15468579530715942]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9713, "test_loss": 0.11475176219940185}, {"step": 3300, "epoch": 7, "train_loss": 0.035091057419776917, "cos_r_negg": [0.388364315032959, 0.44012635946273804, 0.4620133638381958], "cos_innovation_negg": [0.388364315032959, 0.44012635946273804, 0.4620133638381958], "cos_apical_negg": [-0.00514238839969039, 0.03361807018518448, 0.10036838054656982], "cos_Ac_negg": [0.5815119743347168, 0.6519482135772705, 0.6196298599243164], "r_norm": [0.15613043308258057, 0.15476371347904205, 0.155074805021286]}, {"step": 3400, "epoch": 7, "train_loss": 0.018217364326119423, "cos_r_negg": [0.350436806678772, 0.4000297784805298, 0.42972588539123535], "cos_innovation_negg": [0.350436806678772, 0.4000297784805298, 0.42972588539123535], "cos_apical_negg": [1.7241574823856354e-05, 0.03882444649934769, 0.10841445624828339], "cos_Ac_negg": [0.5855284333229065, 0.6549354195594788, 0.6250629425048828], "r_norm": [0.15655122697353363, 0.1532667875289917, 0.15430009365081787]}, {"step": 3500, "epoch": 7, "train_loss": 0.006485956255346537, "cos_r_negg": [0.34565138816833496, 0.39090925455093384, 0.42413222789764404], "cos_innovation_negg": [0.34565138816833496, 0.39090925455093384, 0.42413222789764404], "cos_apical_negg": [0.00023910123854875565, 0.03469627723097801, 0.1001327633857727], "cos_Ac_negg": [0.5923053622245789, 0.6582446098327637, 0.6333015561103821], "r_norm": [0.12530985474586487, 0.12307082861661911, 0.1236409991979599], "ldr": {"dL_sdil": -0.0033531971275806427, "dL_bp": -0.004545159637928009, "ratio": 0.7377512331138399}}, {"step": 3600, "epoch": 7, "train_loss": 0.0025435287971049547, "cos_r_negg": [0.361345112323761, 0.41499537229537964, 0.44833433628082275], "cos_innovation_negg": [0.361345112323761, 0.41499537229537964, 0.44833433628082275], "cos_apical_negg": [0.008182194083929062, 0.04040120542049408, 0.10462453961372375], "cos_Ac_negg": [0.583110511302948, 0.660904586315155, 0.6398817300796509], "r_norm": [0.14871007204055786, 0.14299443364143372, 0.14446698129177094]}, {"step": 3700, "epoch": 7, "train_loss": 0.022246941924095154, "cos_r_negg": [0.36678579449653625, 0.4257878065109253, 0.46085524559020996], "cos_innovation_negg": [0.36678579449653625, 0.4257878065109253, 0.46085524559020996], "cos_apical_negg": [0.010286729782819748, 0.0447201170027256, 0.10164155811071396], "cos_Ac_negg": [0.588095486164093, 0.6620072722434998, 0.6441003084182739], "r_norm": [0.17962360382080078, 0.17733833193778992, 0.17752781510353088]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9726, "test_loss": 0.11701411352157592}, {"step": 3800, "epoch": 8, "train_loss": 0.0020683319307863712, "cos_r_negg": [0.31483468413352966, 0.36443912982940674, 0.41338908672332764], "cos_innovation_negg": [0.31483468413352966, 0.36443912982940674, 0.41338908672332764], "cos_apical_negg": [-0.005256372503936291, 0.03418785333633423, 0.09913021326065063], "cos_Ac_negg": [0.5751336216926575, 0.6563472747802734, 0.6401858329772949], "r_norm": [0.1426611691713333, 0.1388171762228012, 0.13953682780265808]}, {"step": 3900, "epoch": 8, "train_loss": 0.003269444452598691, "cos_r_negg": [0.30942264199256897, 0.3527880609035492, 0.4018915593624115], "cos_innovation_negg": [0.30942264199256897, 0.3527880609035492, 0.4018915593624115], "cos_apical_negg": [0.0003192746080458164, 0.03188473731279373, 0.10008475184440613], "cos_Ac_negg": [0.5807934999465942, 0.656278133392334, 0.6394872665405273], "r_norm": [0.13429084420204163, 0.13071230053901672, 0.1315767467021942]}, {"step": 4000, "epoch": 8, "train_loss": 0.004132644273340702, "cos_r_negg": [0.29055094718933105, 0.33638814091682434, 0.3855329155921936], "cos_innovation_negg": [0.29055094718933105, 0.33638814091682434, 0.3855329155921936], "cos_apical_negg": [-0.00855519250035286, 0.033741869032382965, 0.09871513396501541], "cos_Ac_negg": [0.58148592710495, 0.6534676551818848, 0.644256055355072], "r_norm": [0.1311567723751068, 0.12831135094165802, 0.12942379713058472], "ldr": {"dL_sdil": -0.004478953778743744, "dL_bp": -0.006740003824234009, "ratio": 0.6645328245422428}}, {"step": 4100, "epoch": 8, "train_loss": 0.029480300843715668, "cos_r_negg": [0.32144200801849365, 0.35947489738464355, 0.40849611163139343], "cos_innovation_negg": [0.32144200801849365, 0.35947489738464355, 0.40849611163139343], "cos_apical_negg": [0.003794255666434765, 0.032217852771282196, 0.09655633568763733], "cos_Ac_negg": [0.5934809446334839, 0.6684763431549072, 0.6491529941558838], "r_norm": [0.13236254453659058, 0.12935969233512878, 0.1306191086769104]}, {"step": 4200, "epoch": 8, "train_loss": 0.02441762574017048, "cos_r_negg": [0.31035399436950684, 0.3590101897716522, 0.4074493646621704], "cos_innovation_negg": [0.31035399436950684, 0.3590101897716522, 0.4074493646621704], "cos_apical_negg": [-0.011325972154736519, 0.03502793610095978, 0.10148602724075317], "cos_Ac_negg": [0.5924097299575806, 0.6642954349517822, 0.6504828929901123], "r_norm": [0.13863743841648102, 0.13571245968341827, 0.13672906160354614]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.972, "test_loss": 0.12872028541564942}, {"step": 4300, "epoch": 9, "train_loss": 0.004121402744203806, "cos_r_negg": [0.27059096097946167, 0.31552380323410034, 0.37013643980026245], "cos_innovation_negg": [0.27059096097946167, 0.31552380323410034, 0.37013643980026245], "cos_apical_negg": [-0.008882439695298672, 0.037799298763275146, 0.099274642765522], "cos_Ac_negg": [0.5838530659675598, 0.6564987897872925, 0.6464356184005737], "r_norm": [0.1464133858680725, 0.14511489868164062, 0.14471206068992615]}, {"step": 4400, "epoch": 9, "train_loss": 0.0022535943426191807, "cos_r_negg": [0.2896331548690796, 0.3270096778869629, 0.37870895862579346], "cos_innovation_negg": [0.2896331548690796, 0.3270096778869629, 0.37870895862579346], "cos_apical_negg": [-0.01004791259765625, 0.03820240497589111, 0.09683576226234436], "cos_Ac_negg": [0.5843517780303955, 0.6612729430198669, 0.6517888903617859], "r_norm": [0.14669322967529297, 0.14385291934013367, 0.14458304643630981]}, {"step": 4500, "epoch": 9, "train_loss": 0.03937598317861557, "cos_r_negg": [0.25708484649658203, 0.2989466190338135, 0.3525906801223755], "cos_innovation_negg": [0.25708484649658203, 0.2989466190338135, 0.3525906801223755], "cos_apical_negg": [-0.014484313316643238, 0.03541833162307739, 0.09813150763511658], "cos_Ac_negg": [0.5815752148628235, 0.656388521194458, 0.6512470245361328], "r_norm": [0.14379392564296722, 0.14149987697601318, 0.14191201329231262], "ldr": {"dL_sdil": -0.005707666277885437, "dL_bp": -0.008829593658447266, "ratio": 0.6464245693146838}}, {"step": 4600, "epoch": 9, "train_loss": 0.00029099496896378696, "cos_r_negg": [0.26509222388267517, 0.3011626601219177, 0.35187166929244995], "cos_innovation_negg": [0.26509222388267517, 0.3011626601219177, 0.35187166929244995], "cos_apical_negg": [-0.009409554302692413, 0.02760307490825653, 0.08916454017162323], "cos_Ac_negg": [0.5758092403411865, 0.6568939685821533, 0.6504249572753906], "r_norm": [0.12511903047561646, 0.12319760769605637, 0.12391689419746399]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9741, "test_loss": 0.1214785472869873}, {"step": 4700, "epoch": 10, "train_loss": 0.003846376668661833, "cos_r_negg": [0.24793319404125214, 0.2752615809440613, 0.32855188846588135], "cos_innovation_negg": [0.24793319404125214, 0.2752615809440613, 0.32855188846588135], "cos_apical_negg": [-0.0168413445353508, 0.03185483440756798, 0.08513405919075012], "cos_Ac_negg": [0.5857955813407898, 0.6577849984169006, 0.6549846529960632], "r_norm": [0.1392510086297989, 0.1374475359916687, 0.1374175101518631]}, {"step": 4800, "epoch": 10, "train_loss": 0.003788778791204095, "cos_r_negg": [0.23982921242713928, 0.26741862297058105, 0.32012176513671875], "cos_innovation_negg": [0.23982921242713928, 0.26741862297058105, 0.32012176513671875], "cos_apical_negg": [-0.020968560129404068, 0.028218358755111694, 0.08401869237422943], "cos_Ac_negg": [0.5756700038909912, 0.6568929553031921, 0.653887152671814], "r_norm": [0.1297290325164795, 0.1271601915359497, 0.1279851645231247]}, {"step": 4900, "epoch": 10, "train_loss": 0.0023530491162091494, "cos_r_negg": [0.22722691297531128, 0.26289501786231995, 0.3097301125526428], "cos_innovation_negg": [0.22722691297531128, 0.26289501786231995, 0.3097301125526428], "cos_apical_negg": [-0.018490899354219437, 0.03279675915837288, 0.08060262352228165], "cos_Ac_negg": [0.577909529209137, 0.6588432788848877, 0.6562414169311523], "r_norm": [0.13232503831386566, 0.13036315143108368, 0.13081206381320953]}, {"step": 5000, "epoch": 10, "train_loss": 0.0003272384637966752, "cos_r_negg": [0.23629596829414368, 0.264797568321228, 0.31595954298973083], "cos_innovation_negg": [0.23629596829414368, 0.264797568321228, 0.31595954298973083], "cos_apical_negg": [-0.018870677798986435, 0.028028279542922974, 0.078565314412117], "cos_Ac_negg": [0.5826868414878845, 0.6588236093521118, 0.657131016254425], "r_norm": [0.12348625063896179, 0.12130117416381836, 0.12188145518302917], "ldr": {"dL_sdil": -0.004463203251361847, "dL_bp": -0.007074527442455292, "ratio": 0.6308835873018882}}, {"step": 5100, "epoch": 10, "train_loss": 0.001783785060979426, "cos_r_negg": [0.22899088263511658, 0.2561159133911133, 0.30505847930908203], "cos_innovation_negg": [0.22899088263511658, 0.2561159133911133, 0.30505847930908203], "cos_apical_negg": [-0.014518175274133682, 0.029250551015138626, 0.0742192268371582], "cos_Ac_negg": [0.5863211154937744, 0.6628580689430237, 0.6601808071136475], "r_norm": [0.13324525952339172, 0.1307089626789093, 0.13108068704605103]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9742, "test_loss": 0.12863891973495484}, {"step": 5200, "epoch": 11, "train_loss": 0.0016999029321596026, "cos_r_negg": [0.22935858368873596, 0.25430065393447876, 0.3051876425743103], "cos_innovation_negg": [0.22935858368873596, 0.25430065393447876, 0.3051876425743103], "cos_apical_negg": [-0.01597149670124054, 0.027901211753487587, 0.07903474569320679], "cos_Ac_negg": [0.5843941569328308, 0.6619303226470947, 0.6641751527786255], "r_norm": [0.13848471641540527, 0.13639108836650848, 0.13692373037338257]}, {"step": 5300, "epoch": 11, "train_loss": 0.00149227911606431, "cos_r_negg": [0.21886737644672394, 0.24021688103675842, 0.2908511161804199], "cos_innovation_negg": [0.21886737644672394, 0.24021688103675842, 0.2908511161804199], "cos_apical_negg": [-0.01306607574224472, 0.0243108831346035, 0.07685942947864532], "cos_Ac_negg": [0.5760036110877991, 0.6580331921577454, 0.6615997552871704], "r_norm": [0.12689343094825745, 0.12522368133068085, 0.1255747675895691]}, {"step": 5400, "epoch": 11, "train_loss": 0.003502150531858206, "cos_r_negg": [0.21088576316833496, 0.23419539630413055, 0.2831454277038574], "cos_innovation_negg": [0.21088576316833496, 0.23419539630413055, 0.2831454277038574], "cos_apical_negg": [-0.014293482527136803, 0.02643945813179016, 0.07471708953380585], "cos_Ac_negg": [0.5800578594207764, 0.661342978477478, 0.6615606546401978], "r_norm": [0.12902571260929108, 0.12745392322540283, 0.1277874857187271]}, {"step": 5500, "epoch": 11, "train_loss": 0.0016093183076009154, "cos_r_negg": [0.2041584849357605, 0.22600063681602478, 0.27503910660743713], "cos_innovation_negg": [0.2041584849357605, 0.22600063681602478, 0.27503910660743713], "cos_apical_negg": [-0.013327575288712978, 0.02545972913503647, 0.07442834228277206], "cos_Ac_negg": [0.5758709907531738, 0.6590155959129333, 0.662075400352478], "r_norm": [0.12717200815677643, 0.1250791996717453, 0.12573327124118805], "ldr": {"dL_sdil": -0.005176179111003876, "dL_bp": -0.00866156816482544, "ratio": 0.5976029989608924}}, {"step": 5600, "epoch": 11, "train_loss": 0.001172092743217945, "cos_r_negg": [0.20233036577701569, 0.2277071177959442, 0.27275943756103516], "cos_innovation_negg": [0.20233036577701569, 0.2277071177959442, 0.27275943756103516], "cos_apical_negg": [-0.018805058673024178, 0.026067454367876053, 0.07017955183982849], "cos_Ac_negg": [0.5761364698410034, 0.6579396724700928, 0.6598615646362305], "r_norm": [0.1238504946231842, 0.12152472138404846, 0.1221206784248352]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.9745, "test_loss": 0.1301448366165161}, {"step": 5700, "epoch": 12, "train_loss": 0.00037419566069729626, "cos_r_negg": [0.19927281141281128, 0.22440919280052185, 0.27297529578208923], "cos_innovation_negg": [0.19927281141281128, 0.22440919280052185, 0.27297529578208923], "cos_apical_negg": [-0.017042972147464752, 0.027279961854219437, 0.07182110846042633], "cos_Ac_negg": [0.5766111612319946, 0.6589412093162537, 0.6626189947128296], "r_norm": [0.1261036992073059, 0.12395627796649933, 0.12456410378217697]}, {"step": 5800, "epoch": 12, "train_loss": 0.0006990813417360187, "cos_r_negg": [0.19310584664344788, 0.21785509586334229, 0.26709112524986267], "cos_innovation_negg": [0.19310584664344788, 0.21785509586334229, 0.26709112524986267], "cos_apical_negg": [-0.01848934218287468, 0.02607933059334755, 0.07075376808643341], "cos_Ac_negg": [0.5727724432945251, 0.6582148671150208, 0.6595107316970825], "r_norm": [0.12188367545604706, 0.11990620195865631, 0.12063635885715485]}, {"step": 5900, "epoch": 12, "train_loss": 0.002051290823146701, "cos_r_negg": [0.21225331723690033, 0.231919527053833, 0.28019171953201294], "cos_innovation_negg": [0.21225331723690033, 0.231919527053833, 0.28019171953201294], "cos_apical_negg": [-0.015212031081318855, 0.02723746933043003, 0.07102951407432556], "cos_Ac_negg": [0.5792114734649658, 0.6632776260375977, 0.66792231798172], "r_norm": [0.13065604865550995, 0.12856750190258026, 0.12947514653205872]}, {"step": 6000, "epoch": 12, "train_loss": 0.00021082983585074544, "cos_r_negg": [0.19337938725948334, 0.2173120081424713, 0.26181572675704956], "cos_innovation_negg": [0.19337938725948334, 0.2173120081424713, 0.26181572675704956], "cos_apical_negg": [-0.02074435167014599, 0.026510538533329964, 0.06835349649190903], "cos_Ac_negg": [0.5737708806991577, 0.6608791351318359, 0.6627746820449829], "r_norm": [0.12671895325183868, 0.1245783269405365, 0.12536340951919556], "ldr": {"dL_sdil": -0.005825906991958618, "dL_bp": -0.010343685746192932, "ratio": 0.5632331777000171}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9745, "test_loss": 0.13520586853027344}, {"step": 6100, "epoch": 13, "train_loss": 0.00114873715210706, "cos_r_negg": [0.18851301074028015, 0.21388106048107147, 0.2559266984462738], "cos_innovation_negg": [0.18851301074028015, 0.21388106048107147, 0.2559266984462738], "cos_apical_negg": [-0.023839516565203667, 0.025731373578310013, 0.06313656270503998], "cos_Ac_negg": [0.5762029886245728, 0.6611396670341492, 0.6640243530273438], "r_norm": [0.12193048000335693, 0.1200208067893982, 0.12074489146471024]}, {"step": 6200, "epoch": 13, "train_loss": 0.0012708634603768587, "cos_r_negg": [0.1813572645187378, 0.2073110044002533, 0.24960248172283173], "cos_innovation_negg": [0.1813572645187378, 0.2073110044002533, 0.24960248172283173], "cos_apical_negg": [-0.023270772770047188, 0.025720421224832535, 0.06304657459259033], "cos_Ac_negg": [0.5741591453552246, 0.6597561240196228, 0.6618976593017578], "r_norm": [0.12066633254289627, 0.11884524673223495, 0.11952510476112366]}, {"step": 6300, "epoch": 13, "train_loss": 0.0007790725794620812, "cos_r_negg": [0.17786607146263123, 0.20206767320632935, 0.24386122822761536], "cos_innovation_negg": [0.17786607146263123, 0.20206767320632935, 0.24386122822761536], "cos_apical_negg": [-0.021800057962536812, 0.02558799646794796, 0.06129763275384903], "cos_Ac_negg": [0.5735625624656677, 0.6590381860733032, 0.659139096736908], "r_norm": [0.11918114870786667, 0.11713533848524094, 0.11786467581987381]}, {"step": 6400, "epoch": 13, "train_loss": 0.0002151842345483601, "cos_r_negg": [0.1802089810371399, 0.2039659321308136, 0.24680691957473755], "cos_innovation_negg": [0.1802089810371399, 0.2039659321308136, 0.24680691957473755], "cos_apical_negg": [-0.019582778215408325, 0.026868212968111038, 0.06381715834140778], "cos_Ac_negg": [0.5758476853370667, 0.660313069820404, 0.6641278266906738], "r_norm": [0.12172451615333557, 0.12006828933954239, 0.12067775428295135]}, {"step": 6500, "epoch": 13, "train_loss": 0.0001270119973924011, "cos_r_negg": [0.17689760029315948, 0.1998426914215088, 0.24155038595199585], "cos_innovation_negg": [0.17689760029315948, 0.1998426914215088, 0.24155038595199585], "cos_apical_negg": [-0.02078068070113659, 0.0257948599755764, 0.06032883748412132], "cos_Ac_negg": [0.5716795325279236, 0.6583441495895386, 0.6622625589370728], "r_norm": [0.12071338295936584, 0.11919794976711273, 0.1198136955499649], "ldr": {"dL_sdil": -0.00588168203830719, "dL_bp": -0.010370306670665741, "ratio": 0.5671656803500879}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9749, "test_loss": 0.13689604864120483}, {"step": 6600, "epoch": 14, "train_loss": 0.00015675066970288754, "cos_r_negg": [0.1775672882795334, 0.1971694529056549, 0.2404470443725586], "cos_innovation_negg": [0.1775672882795334, 0.1971694529056549, 0.2404470443725586], "cos_apical_negg": [-0.01920282281935215, 0.025180485099554062, 0.062167614698410034], "cos_Ac_negg": [0.5715060234069824, 0.6585206985473633, 0.6648013591766357], "r_norm": [0.11828052252531052, 0.11654089391231537, 0.11733348667621613]}, {"step": 6700, "epoch": 14, "train_loss": 0.0003808314504567534, "cos_r_negg": [0.17531812191009521, 0.19601497054100037, 0.23779110610485077], "cos_innovation_negg": [0.17531812191009521, 0.19601497054100037, 0.23779110610485077], "cos_apical_negg": [-0.0205191969871521, 0.025503646582365036, 0.06019933521747589], "cos_Ac_negg": [0.5700347423553467, 0.6578106880187988, 0.6626570820808411], "r_norm": [0.11736148595809937, 0.11555007100105286, 0.1163320392370224]}, {"step": 6800, "epoch": 14, "train_loss": 0.0005349187995307148, "cos_r_negg": [0.1725764274597168, 0.1920589953660965, 0.23308731615543365], "cos_innovation_negg": [0.1725764274597168, 0.1920589953660965, 0.23308731615543365], "cos_apical_negg": [-0.019598782062530518, 0.025701753795146942, 0.05938730016350746], "cos_Ac_negg": [0.570386528968811, 0.6572756767272949, 0.6646796464920044], "r_norm": [0.11974599957466125, 0.11808669567108154, 0.11874199658632278]}, {"step": 6900, "epoch": 14, "train_loss": 0.00042516947723925114, "cos_r_negg": [0.16775819659233093, 0.18516004085540771, 0.22745344042778015], "cos_innovation_negg": [0.16775819659233093, 0.18516004085540771, 0.22745344042778015], "cos_apical_negg": [-0.016404155641794205, 0.02493401989340782, 0.05891628563404083], "cos_Ac_negg": [0.569895327091217, 0.6560081243515015, 0.661963701248169], "r_norm": [0.11935018748044968, 0.11756415665149689, 0.11828391253948212]}, {"step": 7000, "epoch": 14, "train_loss": 0.0005088113248348236, "cos_r_negg": [0.1683044284582138, 0.18657205998897552, 0.22744154930114746], "cos_innovation_negg": [0.1683044284582138, 0.18657205998897552, 0.22744154930114746], "cos_apical_negg": [-0.017778633162379265, 0.025177061557769775, 0.05926932394504547], "cos_Ac_negg": [0.5718478560447693, 0.6575223207473755, 0.6637944579124451], "r_norm": [0.11985176056623459, 0.118133045732975, 0.1188424825668335], "ldr": {"dL_sdil": -0.005673840641975403, "dL_bp": -0.010133080184459686, "ratio": 0.5599324725246849}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.9749, "test_loss": 0.13971249618530274}], "final": {"test_acc": 0.9749, "test_loss": 0.13971249618530274, "wall_s": 22.210185527801514, "cos_r_negg": [0.16554266214370728, 0.185797318816185, 0.22594398260116577], "cos_innovation_negg": [0.16554266214370728, 0.185797318816185, 0.22594398260116577], "cos_apical_negg": [-0.019418207928538322, 0.025267083197832108, 0.05846957862377167], "cos_Ac_negg": [0.5687137246131897, 0.6561079621315002, 0.6624050736427307]}} \ No newline at end of file diff --git a/results/nuis_ctrl_v1_rho0p5_residual_s4.json b/results/nuis_ctrl_v1_rho0p5_residual_s4.json new file mode 100644 index 0000000..945a568 --- /dev/null +++ b/results/nuis_ctrl_v1_rho0p5_residual_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "mnist", "depth": 3, "width": 256, "act": "tanh", "residual": 0, "epochs": 15, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.5, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 100, "max_steps": 0, "probe_bs": 512, "outdir": "results", "tag": "nuis_ctrl_v1_rho0p5_residual_s4", "n_in": 784}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4633917808532715, "cos_r_negg": [-0.012780680321156979, 0.029944397509098053, 0.009445806965231895], "cos_innovation_negg": [-0.012780680321156979, 0.029944397509098053, 0.009445806965231895], "cos_apical_negg": [-0.014915822073817253, 0.021098705008625984, 0.001381695969030261], "cos_Ac_negg": [-0.012782195582985878, 0.02994346432387829, 0.00944523699581623], "r_norm": [4.848402500152588, 4.855297088623047, 4.947994709014893], "ldr": {"dL_sdil": -0.003355741500854492, "dL_bp": -0.2672312259674072, "ratio": 0.012557445293701472}}, {"step": 100, "epoch": 0, "train_loss": 0.19629260897636414, "cos_r_negg": [0.45800167322158813, 0.3779235780239105, 0.3199354112148285], "cos_innovation_negg": [0.45800167322158813, 0.3779235780239105, 0.3199354112148285], "cos_apical_negg": [0.0711335837841034, 0.0981348529458046, 0.1037457287311554], "cos_Ac_negg": [0.4605008363723755, 0.37930095195770264, 0.3192102909088135], "r_norm": [0.9201613664627075, 0.9299724102020264, 0.944883406162262]}, {"step": 200, "epoch": 0, "train_loss": 0.2513696253299713, "cos_r_negg": [0.5069111585617065, 0.4413618743419647, 0.35997873544692993], "cos_innovation_negg": [0.5069111585617065, 0.4413618743419647, 0.35997873544692993], "cos_apical_negg": [0.08772313594818115, 0.09376981854438782, 0.11008410155773163], "cos_Ac_negg": [0.5130407810211182, 0.44751471281051636, 0.3597726821899414], "r_norm": [0.7810784578323364, 0.7865335941314697, 0.8042972087860107]}, {"step": 300, "epoch": 0, "train_loss": 0.22510278224945068, "cos_r_negg": [0.5284439921379089, 0.4661669135093689, 0.3702007234096527], "cos_innovation_negg": [0.5284439921379089, 0.4661669135093689, 0.3702007234096527], "cos_apical_negg": [0.06006728857755661, 0.08298870921134949, 0.09690611064434052], "cos_Ac_negg": [0.5321969985961914, 0.4673411548137665, 0.370270699262619], "r_norm": [0.5515317916870117, 0.5567474365234375, 0.5702890753746033]}, {"step": 400, "epoch": 0, "train_loss": 0.11462095379829407, "cos_r_negg": [0.541813313961029, 0.4840790033340454, 0.40467655658721924], "cos_innovation_negg": [0.541813313961029, 0.4840790033340454, 0.40467655658721924], "cos_apical_negg": [0.07068207859992981, 0.08267825841903687, 0.10273025929927826], "cos_Ac_negg": [0.5436958074569702, 0.48474395275115967, 0.40501704812049866], "r_norm": [0.549662709236145, 0.5556418895721436, 0.5674132108688354]}, {"epoch_end": 0, "step": 469, "test_acc": 0.9464, "test_loss": 0.1793723587036133}, {"step": 500, "epoch": 1, "train_loss": 0.16420914232730865, "cos_r_negg": [0.530678391456604, 0.5233479738235474, 0.44211336970329285], "cos_innovation_negg": [0.530678391456604, 0.5233479738235474, 0.44211336970329285], "cos_apical_negg": [0.054169267416000366, 0.09084159135818481, 0.10243886709213257], "cos_Ac_negg": [0.5345008373260498, 0.528390645980835, 0.4451415240764618], "r_norm": [0.5338467359542847, 0.5425035357475281, 0.5567911863327026], "ldr": {"dL_sdil": -0.014890104532241821, "dL_bp": -0.013183802366256714, "ratio": 1.1294241311104831}}, {"step": 600, "epoch": 1, "train_loss": 0.06959572434425354, "cos_r_negg": [0.517897367477417, 0.48799532651901245, 0.4216245114803314], "cos_innovation_negg": [0.517897367477417, 0.48799532651901245, 0.4216245114803314], "cos_apical_negg": [0.06074514985084534, 0.07454491406679153, 0.09456928074359894], "cos_Ac_negg": [0.5248046517372131, 0.4942425489425659, 0.4247208833694458], "r_norm": [0.4789523184299469, 0.4827333390712738, 0.4942224323749542]}, {"step": 700, "epoch": 1, "train_loss": 0.18972012400627136, "cos_r_negg": [0.5590490102767944, 0.5158365964889526, 0.4349728226661682], "cos_innovation_negg": [0.5590490102767944, 0.5158365964889526, 0.4349728226661682], "cos_apical_negg": [0.04988925904035568, 0.07902048528194427, 0.09580082446336746], "cos_Ac_negg": [0.5671764612197876, 0.5219216346740723, 0.43704646825790405], "r_norm": [0.448728084564209, 0.45318329334259033, 0.4638993740081787]}, {"step": 800, "epoch": 1, "train_loss": 0.12382678687572479, "cos_r_negg": [0.5426483154296875, 0.5266801118850708, 0.4465424418449402], "cos_innovation_negg": [0.5426483154296875, 0.5266801118850708, 0.4465424418449402], "cos_apical_negg": [0.031039439141750336, 0.06186395883560181, 0.09613986313343048], "cos_Ac_negg": [0.5485931634902954, 0.5314885973930359, 0.44805508852005005], "r_norm": [0.3445935845375061, 0.34737735986709595, 0.356315940618515]}, {"step": 900, "epoch": 1, "train_loss": 0.23975449800491333, "cos_r_negg": [0.5662952661514282, 0.5434687733650208, 0.4764929413795471], "cos_innovation_negg": [0.5662952661514282, 0.5434687733650208, 0.4764929413795471], "cos_apical_negg": [0.05266273021697998, 0.0776781439781189, 0.11314323544502258], "cos_Ac_negg": [0.571251630783081, 0.5463084578514099, 0.4745595455169678], "r_norm": [0.44510284066200256, 0.4486859440803528, 0.4580800533294678]}, {"epoch_end": 1, "step": 938, "test_acc": 0.952, "test_loss": 0.1661437412261963}, {"step": 1000, "epoch": 2, "train_loss": 0.028905929997563362, "cos_r_negg": [0.554603099822998, 0.5429319739341736, 0.4734734296798706], "cos_innovation_negg": [0.554603099822998, 0.5429319739341736, 0.4734734296798706], "cos_apical_negg": [0.0443001426756382, 0.07058797776699066, 0.10145597159862518], "cos_Ac_negg": [0.5633929967880249, 0.5492414236068726, 0.47602379322052], "r_norm": [0.361183226108551, 0.3640197515487671, 0.37107646465301514], "ldr": {"dL_sdil": -0.009004637598991394, "dL_bp": -0.007804587483406067, "ratio": 1.1537621454224}}, {"step": 1100, "epoch": 2, "train_loss": 0.16364812850952148, "cos_r_negg": [0.5589355230331421, 0.552735447883606, 0.4882117509841919], "cos_innovation_negg": [0.5589355230331421, 0.552735447883606, 0.4882117509841919], "cos_apical_negg": [0.028568606823682785, 0.06963034719228745, 0.10563243180513382], "cos_Ac_negg": [0.575088381767273, 0.5662277936935425, 0.4955238103866577], "r_norm": [0.38412028551101685, 0.39031827449798584, 0.39872923493385315]}, {"step": 1200, "epoch": 2, "train_loss": 0.15252575278282166, "cos_r_negg": [0.5665971040725708, 0.5628219842910767, 0.4772229790687561], "cos_innovation_negg": [0.5665971040725708, 0.5628219842910767, 0.4772229790687561], "cos_apical_negg": [0.03518972918391228, 0.05924806743860245, 0.08513158559799194], "cos_Ac_negg": [0.5841786861419678, 0.5764858722686768, 0.484283983707428], "r_norm": [0.2955557107925415, 0.297440767288208, 0.3046206831932068]}, {"step": 1300, "epoch": 2, "train_loss": 0.0833682268857956, "cos_r_negg": [0.5749170780181885, 0.5683309435844421, 0.48875701427459717], "cos_innovation_negg": [0.5749170780181885, 0.5683309435844421, 0.48875701427459717], "cos_apical_negg": [0.036705683916807175, 0.062310874462127686, 0.0929577648639679], "cos_Ac_negg": [0.5939832925796509, 0.5838811993598938, 0.4961000084877014], "r_norm": [0.3090793192386627, 0.3120633661746979, 0.3191313147544861]}, {"step": 1400, "epoch": 2, "train_loss": 0.14456184208393097, "cos_r_negg": [0.5595439672470093, 0.5631247758865356, 0.4859144687652588], "cos_innovation_negg": [0.5595439672470093, 0.5631247758865356, 0.4859144687652588], "cos_apical_negg": [0.04818922281265259, 0.06211773678660393, 0.08960709720849991], "cos_Ac_negg": [0.5791299343109131, 0.5801452994346619, 0.49707621335983276], "r_norm": [0.277264267206192, 0.27880436182022095, 0.2881801128387451]}, {"epoch_end": 2, "step": 1407, "test_acc": 0.9656, "test_loss": 0.12155494270324707}, {"step": 1500, "epoch": 3, "train_loss": 0.030826527625322342, "cos_r_negg": [0.5728583335876465, 0.5700634717941284, 0.5081031322479248], "cos_innovation_negg": [0.5728583335876465, 0.5700634717941284, 0.5081031322479248], "cos_apical_negg": [0.028567587956786156, 0.05023908615112305, 0.09261767566204071], "cos_Ac_negg": [0.5992041826248169, 0.5973564386367798, 0.5250089168548584], "r_norm": [0.26716792583465576, 0.26799115538597107, 0.2772371172904968], "ldr": {"dL_sdil": -0.005272507667541504, "dL_bp": -0.005541227757930756, "ratio": 0.9515053157660498}}, {"step": 1600, "epoch": 3, "train_loss": 0.05727434903383255, "cos_r_negg": [0.565005898475647, 0.5765089988708496, 0.5215156674385071], "cos_innovation_negg": [0.565005898475647, 0.5765089988708496, 0.5215156674385071], "cos_apical_negg": [0.02152979001402855, 0.06541414558887482, 0.09821660816669464], "cos_Ac_negg": [0.5956593751907349, 0.6005531549453735, 0.5370639562606812], "r_norm": [0.24979785084724426, 0.25126469135284424, 0.25749334692955017]}, {"step": 1700, "epoch": 3, "train_loss": 0.05129840970039368, "cos_r_negg": [0.5355230569839478, 0.5582835674285889, 0.5039918422698975], "cos_innovation_negg": [0.5355230569839478, 0.5582835674285889, 0.5039918422698975], "cos_apical_negg": [0.025196142494678497, 0.059872645884752274, 0.09832684695720673], "cos_Ac_negg": [0.5879979729652405, 0.5970520973205566, 0.5267399549484253], "r_norm": [0.24554318189620972, 0.24536576867103577, 0.2531190812587738]}, {"step": 1800, "epoch": 3, "train_loss": 0.012955398298799992, "cos_r_negg": [0.5604230165481567, 0.5711764693260193, 0.5186675786972046], "cos_innovation_negg": [0.5604230165481567, 0.5711764693260193, 0.5186675786972046], "cos_apical_negg": [0.01740376092493534, 0.06112934648990631, 0.0935158059000969], "cos_Ac_negg": [0.6019397377967834, 0.602321445941925, 0.5405991077423096], "r_norm": [0.2344096601009369, 0.23512639105319977, 0.24174676835536957]}, {"epoch_end": 3, "step": 1876, "test_acc": 0.9671, "test_loss": 0.11272891120910644}, {"step": 1900, "epoch": 4, "train_loss": 0.058084309101104736, "cos_r_negg": [0.5556366443634033, 0.5862893462181091, 0.5184904336929321], "cos_innovation_negg": [0.5556366443634033, 0.5862893462181091, 0.5184904336929321], "cos_apical_negg": [0.015282210893929005, 0.06368627399206161, 0.09846051037311554], "cos_Ac_negg": [0.5857324600219727, 0.615715503692627, 0.5340960025787354], "r_norm": [0.24607497453689575, 0.244879812002182, 0.2525624632835388]}, {"step": 2000, "epoch": 4, "train_loss": 0.1338292509317398, "cos_r_negg": [0.5271744728088379, 0.5458658933639526, 0.5023095607757568], "cos_innovation_negg": [0.5271744728088379, 0.5458658933639526, 0.5023095607757568], "cos_apical_negg": [0.008943749591708183, 0.04826200753450394, 0.0869932621717453], "cos_Ac_negg": [0.595009446144104, 0.6109379529953003, 0.5437473058700562], "r_norm": [0.18371659517288208, 0.18403282761573792, 0.18818703293800354], "ldr": {"dL_sdil": -0.004323810338973999, "dL_bp": -0.004415325820446014, "ratio": 0.9792732212313222}}, {"step": 2100, "epoch": 4, "train_loss": 0.0427069216966629, "cos_r_negg": [0.5470823049545288, 0.5737941861152649, 0.5336804389953613], "cos_innovation_negg": [0.5470823049545288, 0.5737941861152649, 0.5336804389953613], "cos_apical_negg": [0.03122435137629509, 0.05551818385720253, 0.10068430006504059], "cos_Ac_negg": [0.5934459567070007, 0.6172520518302917, 0.5573433637619019], "r_norm": [0.2578936815261841, 0.2589249610900879, 0.26507359743118286]}, {"step": 2200, "epoch": 4, "train_loss": 0.007049845065921545, "cos_r_negg": [0.5366842746734619, 0.5685009956359863, 0.5282079577445984], "cos_innovation_negg": [0.5366842746734619, 0.5685009956359863, 0.5282079577445984], "cos_apical_negg": [0.0284306351095438, 0.05532219260931015, 0.09404999017715454], "cos_Ac_negg": [0.5955783724784851, 0.6221433281898499, 0.554222583770752], "r_norm": [0.18503259122371674, 0.18524736166000366, 0.1902545988559723]}, {"step": 2300, "epoch": 4, "train_loss": 0.16544701159000397, "cos_r_negg": [0.5196713209152222, 0.5407743453979492, 0.5175164937973022], "cos_innovation_negg": [0.5196713209152222, 0.5407743453979492, 0.5175164937973022], "cos_apical_negg": [0.028797149658203125, 0.05777539312839508, 0.0995645523071289], "cos_Ac_negg": [0.597223162651062, 0.6164929270744324, 0.5585464835166931], "r_norm": [0.21977999806404114, 0.22056253254413605, 0.22670689225196838]}, {"epoch_end": 4, "step": 2345, "test_acc": 0.9688, "test_loss": 0.1113053092956543}, {"step": 2400, "epoch": 5, "train_loss": 0.10427281260490417, "cos_r_negg": [0.5354389548301697, 0.5584008693695068, 0.5256493091583252], "cos_innovation_negg": [0.5354389548301697, 0.5584008693695068, 0.5256493091583252], "cos_apical_negg": [0.01502544991672039, 0.05919471010565758, 0.1018826812505722], "cos_Ac_negg": [0.6240832805633545, 0.6391306519508362, 0.5698639154434204], "r_norm": [0.19956517219543457, 0.19906964898109436, 0.20437994599342346]}, {"step": 2500, "epoch": 5, "train_loss": 0.03229810297489166, "cos_r_negg": [0.4793948531150818, 0.49479731917381287, 0.48169025778770447], "cos_innovation_negg": [0.4793948531150818, 0.49479731917381287, 0.48169025778770447], "cos_apical_negg": [0.005698446650058031, 0.042566392570734024, 0.09249535202980042], "cos_Ac_negg": [0.5997303128242493, 0.6236845254898071, 0.56367427110672], "r_norm": [0.19791555404663086, 0.19657497107982635, 0.20220273733139038], "ldr": {"dL_sdil": -0.00717780739068985, "dL_bp": -0.00806998461484909, "ratio": 0.8894449906982971}}, {"step": 2600, "epoch": 5, "train_loss": 0.07370933890342712, "cos_r_negg": [0.4796818792819977, 0.5058951377868652, 0.4986645579338074], "cos_innovation_negg": [0.4796818792819977, 0.5058951377868652, 0.4986645579338074], "cos_apical_negg": [0.011753657832741737, 0.04593665152788162, 0.10054924339056015], "cos_Ac_negg": [0.6071487665176392, 0.6265208721160889, 0.5665008425712585], "r_norm": [0.2014935314655304, 0.20205116271972656, 0.20769213140010834]}, {"step": 2700, "epoch": 5, "train_loss": 0.029471423476934433, "cos_r_negg": [0.4960041642189026, 0.5312337875366211, 0.5180087089538574], "cos_innovation_negg": [0.4960041642189026, 0.5312337875366211, 0.5180087089538574], "cos_apical_negg": [0.00802630465477705, 0.04351513832807541, 0.10110777616500854], "cos_Ac_negg": [0.6111445426940918, 0.6398720145225525, 0.5836657285690308], "r_norm": [0.18545469641685486, 0.18432360887527466, 0.19030028581619263]}, {"step": 2800, "epoch": 5, "train_loss": 0.06277310848236084, "cos_r_negg": [0.49087902903556824, 0.52158522605896, 0.5169394016265869], "cos_innovation_negg": [0.49087902903556824, 0.52158522605896, 0.5169394016265869], "cos_apical_negg": [0.0025420268066227436, 0.04631838947534561, 0.10575099289417267], "cos_Ac_negg": [0.6145914793014526, 0.6398005485534668, 0.577639102935791], "r_norm": [0.1873975694179535, 0.18691596388816833, 0.19314013421535492]}, {"epoch_end": 5, "step": 2814, "test_acc": 0.9695, "test_loss": 0.11885701427459716}, {"step": 2900, "epoch": 6, "train_loss": 0.010572376661002636, "cos_r_negg": [0.48112568259239197, 0.5033720135688782, 0.5046048760414124], "cos_innovation_negg": [0.48112568259239197, 0.5033720135688782, 0.5046048760414124], "cos_apical_negg": [0.010897192172706127, 0.04461832344532013, 0.10347127914428711], "cos_Ac_negg": [0.6149629950523376, 0.639897346496582, 0.5943949222564697], "r_norm": [0.1759517341852188, 0.17655286192893982, 0.18292826414108276]}, {"step": 3000, "epoch": 6, "train_loss": 0.06422223895788193, "cos_r_negg": [0.448468416929245, 0.47519180178642273, 0.4838230609893799], "cos_innovation_negg": [0.448468416929245, 0.47519180178642273, 0.4838230609893799], "cos_apical_negg": [0.010866823606193066, 0.043390341103076935, 0.09894081205129623], "cos_Ac_negg": [0.6178638935089111, 0.6467363834381104, 0.5909932851791382], "r_norm": [0.19284075498580933, 0.19198626279830933, 0.19926923513412476], "ldr": {"dL_sdil": -0.005936957895755768, "dL_bp": -0.007723607122898102, "ratio": 0.7686768372972426}}, {"step": 3100, "epoch": 6, "train_loss": 0.006425193510949612, "cos_r_negg": [0.4407464861869812, 0.4647056460380554, 0.4764006733894348], "cos_innovation_negg": [0.4407464861869812, 0.4647056460380554, 0.4764006733894348], "cos_apical_negg": [0.008207781240344048, 0.03860614448785782, 0.1031840592622757], "cos_Ac_negg": [0.6172384023666382, 0.6485732197761536, 0.5958876013755798], "r_norm": [0.17360325157642365, 0.1732999086380005, 0.17874491214752197]}, {"step": 3200, "epoch": 6, "train_loss": 0.017726633697748184, "cos_r_negg": [0.4194863736629486, 0.4550021290779114, 0.4600214958190918], "cos_innovation_negg": [0.4194863736629486, 0.4550021290779114, 0.4600214958190918], "cos_apical_negg": [0.010972080752253532, 0.03930789232254028, 0.10835323482751846], "cos_Ac_negg": [0.6048999428749084, 0.6467569470405579, 0.5933388471603394], "r_norm": [0.16811835765838623, 0.16740605235099792, 0.174017533659935]}, {"epoch_end": 6, "step": 3283, "test_acc": 0.9698, "test_loss": 0.11931576595306397}, {"step": 3300, "epoch": 7, "train_loss": 0.00721763214096427, "cos_r_negg": [0.4085007905960083, 0.44635307788848877, 0.4591672420501709], "cos_innovation_negg": [0.4085007905960083, 0.44635307788848877, 0.4591672420501709], "cos_apical_negg": [0.014309000223875046, 0.03784601762890816, 0.10075211524963379], "cos_Ac_negg": [0.5997157096862793, 0.6449728012084961, 0.5950531959533691], "r_norm": [0.17923204600811005, 0.17848458886146545, 0.1849614530801773]}, {"step": 3400, "epoch": 7, "train_loss": 0.026761548593640327, "cos_r_negg": [0.3779003620147705, 0.41167864203453064, 0.4369514584541321], "cos_innovation_negg": [0.3779003620147705, 0.41167864203453064, 0.4369514584541321], "cos_apical_negg": [0.013029232621192932, 0.03887690231204033, 0.10227739065885544], "cos_Ac_negg": [0.6083661913871765, 0.6476078629493713, 0.5979009866714478], "r_norm": [0.181646466255188, 0.1802097111940384, 0.18684522807598114]}, {"step": 3500, "epoch": 7, "train_loss": 0.006954504176974297, "cos_r_negg": [0.3759863078594208, 0.4159795045852661, 0.43898457288742065], "cos_innovation_negg": [0.3759863078594208, 0.4159795045852661, 0.43898457288742065], "cos_apical_negg": [0.0113754253834486, 0.037376537919044495, 0.09893661737442017], "cos_Ac_negg": [0.6095080375671387, 0.6565811038017273, 0.6077688932418823], "r_norm": [0.173026904463768, 0.17212706804275513, 0.1790306270122528], "ldr": {"dL_sdil": -0.006458871066570282, "dL_bp": -0.00925774872303009, "ratio": 0.6976718919258238}}, {"step": 3600, "epoch": 7, "train_loss": 0.0026093125343322754, "cos_r_negg": [0.38745319843292236, 0.4301786720752716, 0.4510011076927185], "cos_innovation_negg": [0.38745319843292236, 0.4301786720752716, 0.4510011076927185], "cos_apical_negg": [0.01292348001152277, 0.04219462350010872, 0.10134126991033554], "cos_Ac_negg": [0.6150903105735779, 0.6579806804656982, 0.6177340745925903], "r_norm": [0.20709224045276642, 0.20842325687408447, 0.21559664607048035]}, {"step": 3700, "epoch": 7, "train_loss": 0.019069042056798935, "cos_r_negg": [0.37902718782424927, 0.4183151125907898, 0.4412444829940796], "cos_innovation_negg": [0.37902718782424927, 0.4183151125907898, 0.4412444829940796], "cos_apical_negg": [0.0022362112067639828, 0.04025723412632942, 0.09762631356716156], "cos_Ac_negg": [0.6108595728874207, 0.6559834480285645, 0.6185678243637085], "r_norm": [0.1758161336183548, 0.17363697290420532, 0.18072159588336945]}, {"epoch_end": 7, "step": 3752, "test_acc": 0.9711, "test_loss": 0.12247285003662109}, {"step": 3800, "epoch": 8, "train_loss": 0.003969566430896521, "cos_r_negg": [0.34392043948173523, 0.3908594250679016, 0.41961246728897095], "cos_innovation_negg": [0.34392043948173523, 0.3908594250679016, 0.41961246728897095], "cos_apical_negg": [-0.003374006599187851, 0.04298056662082672, 0.10216223448514938], "cos_Ac_negg": [0.6037557125091553, 0.6528016328811646, 0.6152772903442383], "r_norm": [0.17288324236869812, 0.17106732726097107, 0.17709693312644958]}, {"step": 3900, "epoch": 8, "train_loss": 0.01567823626101017, "cos_r_negg": [0.3158973753452301, 0.35698381066322327, 0.39538025856018066], "cos_innovation_negg": [0.3158973753452301, 0.35698381066322327, 0.39538025856018066], "cos_apical_negg": [0.00018017785623669624, 0.034361060708761215, 0.09909682720899582], "cos_Ac_negg": [0.602450966835022, 0.6537513136863708, 0.6154896020889282], "r_norm": [0.15735168755054474, 0.15631639957427979, 0.16174012422561646]}, {"step": 4000, "epoch": 8, "train_loss": 0.0019326929468661547, "cos_r_negg": [0.28406190872192383, 0.33177649974823, 0.3735637366771698], "cos_innovation_negg": [0.28406190872192383, 0.33177649974823, 0.3735637366771698], "cos_apical_negg": [-0.0009207772091031075, 0.033045124262571335, 0.09716273099184036], "cos_Ac_negg": [0.5954470634460449, 0.6512999534606934, 0.6152539253234863], "r_norm": [0.16715312004089355, 0.1667403280735016, 0.17308004200458527], "ldr": {"dL_sdil": -0.006554506719112396, "dL_bp": -0.009801022708415985, "ratio": 0.6687574260474005}}, {"step": 4100, "epoch": 8, "train_loss": 0.007003558799624443, "cos_r_negg": [0.3262624144554138, 0.35910630226135254, 0.39081913232803345], "cos_innovation_negg": [0.3262624144554138, 0.35910630226135254, 0.39081913232803345], "cos_apical_negg": [-0.001367403194308281, 0.03835854306817055, 0.1013665571808815], "cos_Ac_negg": [0.6056264042854309, 0.6569233536720276, 0.6184529662132263], "r_norm": [0.16894903779029846, 0.16885387897491455, 0.1749841719865799]}, {"step": 4200, "epoch": 8, "train_loss": 0.010813431814312935, "cos_r_negg": [0.2951622009277344, 0.33320116996765137, 0.36646828055381775], "cos_innovation_negg": [0.2951622009277344, 0.33320116996765137, 0.36646828055381775], "cos_apical_negg": [-0.0024127052165567875, 0.038089122623205185, 0.09396514296531677], "cos_Ac_negg": [0.5953919291496277, 0.6543108224868774, 0.62578284740448], "r_norm": [0.16695305705070496, 0.16603316366672516, 0.17283734679222107]}, {"epoch_end": 8, "step": 4221, "test_acc": 0.9706, "test_loss": 0.1292420503616333}, {"step": 4300, "epoch": 9, "train_loss": 0.002269454300403595, "cos_r_negg": [0.2853457033634186, 0.3161616325378418, 0.3536715507507324], "cos_innovation_negg": [0.2853457033634186, 0.3161616325378418, 0.3536715507507324], "cos_apical_negg": [0.0022399649024009705, 0.03770298883318901, 0.09852631390094757], "cos_Ac_negg": [0.6070721745491028, 0.6610793471336365, 0.627305269241333], "r_norm": [0.17605867981910706, 0.17580771446228027, 0.18151262402534485]}, {"step": 4400, "epoch": 9, "train_loss": 0.000984566519036889, "cos_r_negg": [0.28093087673187256, 0.3156611919403076, 0.3515191078186035], "cos_innovation_negg": [0.28093087673187256, 0.3156611919403076, 0.3515191078186035], "cos_apical_negg": [0.0019782809540629387, 0.0331115648150444, 0.0966448113322258], "cos_Ac_negg": [0.6000696420669556, 0.6561236381530762, 0.6260621547698975], "r_norm": [0.15039901435375214, 0.1497570425271988, 0.15484662353992462]}, {"step": 4500, "epoch": 9, "train_loss": 0.04025477170944214, "cos_r_negg": [0.25745469331741333, 0.29918572306632996, 0.33587926626205444], "cos_innovation_negg": [0.25745469331741333, 0.29918572306632996, 0.33587926626205444], "cos_apical_negg": [-0.0025087075773626566, 0.034905947744846344, 0.0992714911699295], "cos_Ac_negg": [0.595043420791626, 0.6595293879508972, 0.6286614537239075], "r_norm": [0.1659528911113739, 0.1653435230255127, 0.1706428825855255], "ldr": {"dL_sdil": -0.008124127984046936, "dL_bp": -0.015337422490119934, "ratio": 0.5296931729748163}}, {"step": 4600, "epoch": 9, "train_loss": 0.0028801611624658108, "cos_r_negg": [0.2574304938316345, 0.3032705783843994, 0.3444207012653351], "cos_innovation_negg": [0.2574304938316345, 0.3032705783843994, 0.3444207012653351], "cos_apical_negg": [-0.005891783162951469, 0.034201815724372864, 0.09854810684919357], "cos_Ac_negg": [0.597135603427887, 0.6575795412063599, 0.6337478160858154], "r_norm": [0.1643073856830597, 0.16282491385936737, 0.16824829578399658]}, {"epoch_end": 9, "step": 4690, "test_acc": 0.9722, "test_loss": 0.1323427827835083}, {"step": 4700, "epoch": 10, "train_loss": 0.0009370993357151747, "cos_r_negg": [0.24626906216144562, 0.2893257141113281, 0.32780081033706665], "cos_innovation_negg": [0.24626906216144562, 0.2893257141113281, 0.32780081033706665], "cos_apical_negg": [-0.005222572945058346, 0.03438138961791992, 0.09457738697528839], "cos_Ac_negg": [0.5913804173469543, 0.6587918400764465, 0.6338762044906616], "r_norm": [0.15964040160179138, 0.15823058784008026, 0.16422130167484283]}, {"step": 4800, "epoch": 10, "train_loss": 0.0007197074592113495, "cos_r_negg": [0.23458249866962433, 0.2753637135028839, 0.31380969285964966], "cos_innovation_negg": [0.23458249866962433, 0.2753637135028839, 0.31380969285964966], "cos_apical_negg": [-0.005553705617785454, 0.03434283286333084, 0.09017827361822128], "cos_Ac_negg": [0.5894259214401245, 0.6574665904045105, 0.6380923986434937], "r_norm": [0.15573932230472565, 0.15402621030807495, 0.16002517938613892]}, {"step": 4900, "epoch": 10, "train_loss": 0.0013206034200266004, "cos_r_negg": [0.23630955815315247, 0.27050358057022095, 0.3087467551231384], "cos_innovation_negg": [0.23630955815315247, 0.27050358057022095, 0.3087467551231384], "cos_apical_negg": [-0.0024220035411417484, 0.03346005454659462, 0.08858700096607208], "cos_Ac_negg": [0.5860074162483215, 0.655839204788208, 0.6307581067085266], "r_norm": [0.1568376123905182, 0.15460513532161713, 0.16078686714172363]}, {"step": 5000, "epoch": 10, "train_loss": 0.0007374219130724669, "cos_r_negg": [0.23427411913871765, 0.2656172811985016, 0.3014724850654602], "cos_innovation_negg": [0.23427411913871765, 0.2656172811985016, 0.3014724850654602], "cos_apical_negg": [0.0003635631874203682, 0.03415030986070633, 0.08427981287240982], "cos_Ac_negg": [0.5873151421546936, 0.6557784676551819, 0.6347193717956543], "r_norm": [0.14998020231723785, 0.14845165610313416, 0.1539243459701538], "ldr": {"dL_sdil": -0.006916850805282593, "dL_bp": -0.012052781879901886, "ratio": 0.5738800282129471}}, {"step": 5100, "epoch": 10, "train_loss": 0.005760518833994865, "cos_r_negg": [0.23182448744773865, 0.2668743133544922, 0.30623435974121094], "cos_innovation_negg": [0.23182448744773865, 0.2668743133544922, 0.30623435974121094], "cos_apical_negg": [-0.007979713380336761, 0.033316440880298615, 0.08807151764631271], "cos_Ac_negg": [0.5932509303092957, 0.659389317035675, 0.6369228363037109], "r_norm": [0.1574036180973053, 0.1557951420545578, 0.16126936674118042]}, {"epoch_end": 10, "step": 5159, "test_acc": 0.9729, "test_loss": 0.1333346004486084}, {"step": 5200, "epoch": 11, "train_loss": 0.0006942336913198233, "cos_r_negg": [0.22772935032844543, 0.25820639729499817, 0.2920970022678375], "cos_innovation_negg": [0.22772935032844543, 0.25820639729499817, 0.2920970022678375], "cos_apical_negg": [-0.004930439405143261, 0.03170959651470184, 0.08223657310009003], "cos_Ac_negg": [0.5905312299728394, 0.6572005748748779, 0.6335728168487549], "r_norm": [0.15343058109283447, 0.1519003063440323, 0.15699033439159393]}, {"step": 5300, "epoch": 11, "train_loss": 0.0007566719432361424, "cos_r_negg": [0.21925725042819977, 0.24757423996925354, 0.28562384843826294], "cos_innovation_negg": [0.21925725042819977, 0.24757423996925354, 0.28562384843826294], "cos_apical_negg": [-0.0033702433574944735, 0.029400639235973358, 0.08285006135702133], "cos_Ac_negg": [0.5863053202629089, 0.6559584140777588, 0.6342511177062988], "r_norm": [0.15561218559741974, 0.15404945611953735, 0.15954509377479553]}, {"step": 5400, "epoch": 11, "train_loss": 0.001521232188679278, "cos_r_negg": [0.20712414383888245, 0.23623737692832947, 0.2748767137527466], "cos_innovation_negg": [0.20712414383888245, 0.23623737692832947, 0.2748767137527466], "cos_apical_negg": [-0.005037623457610607, 0.027977438643574715, 0.07745343446731567], "cos_Ac_negg": [0.5803042650222778, 0.6506218910217285, 0.6329739689826965], "r_norm": [0.15041325986385345, 0.14898334443569183, 0.15425938367843628]}, {"step": 5500, "epoch": 11, "train_loss": 0.0026994990184903145, "cos_r_negg": [0.20781847834587097, 0.23885568976402283, 0.2764872908592224], "cos_innovation_negg": [0.20781847834587097, 0.23885568976402283, 0.2764872908592224], "cos_apical_negg": [-0.004095473326742649, 0.030003715306520462, 0.0819830596446991], "cos_Ac_negg": [0.5818891525268555, 0.6552667617797852, 0.6381473541259766], "r_norm": [0.15286672115325928, 0.15100723505020142, 0.1567622721195221], "ldr": {"dL_sdil": -0.007533937692642212, "dL_bp": -0.013635903596878052, "ratio": 0.5525074036433575}}, {"step": 5600, "epoch": 11, "train_loss": 0.0021804312709718943, "cos_r_negg": [0.1956554353237152, 0.22334757447242737, 0.2606690227985382], "cos_innovation_negg": [0.1956554353237152, 0.22334757447242737, 0.2606690227985382], "cos_apical_negg": [-0.005527795292437077, 0.027708079665899277, 0.07460241764783859], "cos_Ac_negg": [0.580459475517273, 0.6528034210205078, 0.6360132694244385], "r_norm": [0.14966024458408356, 0.1478385031223297, 0.15374064445495605]}, {"epoch_end": 11, "step": 5628, "test_acc": 0.973, "test_loss": 0.13941197929382323}, {"step": 5700, "epoch": 12, "train_loss": 0.0010215011425316334, "cos_r_negg": [0.19373255968093872, 0.22226883471012115, 0.25930389761924744], "cos_innovation_negg": [0.19373255968093872, 0.22226883471012115, 0.25930389761924744], "cos_apical_negg": [-0.0043902210891246796, 0.029867716133594513, 0.0760422945022583], "cos_Ac_negg": [0.579633891582489, 0.6534334421157837, 0.635990560054779], "r_norm": [0.1503656506538391, 0.14871734380722046, 0.15430158376693726]}, {"step": 5800, "epoch": 12, "train_loss": 0.00032125189318321645, "cos_r_negg": [0.18696382641792297, 0.2155623733997345, 0.2532934546470642], "cos_innovation_negg": [0.18696382641792297, 0.2155623733997345, 0.2532934546470642], "cos_apical_negg": [-0.005205532535910606, 0.029154784977436066, 0.07369737327098846], "cos_Ac_negg": [0.577888011932373, 0.6530437469482422, 0.6363422870635986], "r_norm": [0.1520848274230957, 0.1502857506275177, 0.15606342256069183]}, {"step": 5900, "epoch": 12, "train_loss": 0.0010295454412698746, "cos_r_negg": [0.186716690659523, 0.21536558866500854, 0.25176239013671875], "cos_innovation_negg": [0.186716690659523, 0.21536558866500854, 0.25176239013671875], "cos_apical_negg": [-0.0039062260184437037, 0.030423138290643692, 0.07383649796247482], "cos_Ac_negg": [0.5770515203475952, 0.6527430415153503, 0.6366908550262451], "r_norm": [0.15548017621040344, 0.15395236015319824, 0.15984785556793213]}, {"step": 6000, "epoch": 12, "train_loss": 0.00032391524291597307, "cos_r_negg": [0.18734830617904663, 0.21532535552978516, 0.2516634464263916], "cos_innovation_negg": [0.18734830617904663, 0.21532535552978516, 0.2516634464263916], "cos_apical_negg": [-0.005374058149755001, 0.029912449419498444, 0.07371838390827179], "cos_Ac_negg": [0.57501220703125, 0.6550776362419128, 0.638787031173706], "r_norm": [0.15264374017715454, 0.15102964639663696, 0.15677979588508606], "ldr": {"dL_sdil": -0.008371993899345398, "dL_bp": -0.015956230461597443, "ratio": 0.5246849448241953}}, {"epoch_end": 12, "step": 6097, "test_acc": 0.9732, "test_loss": 0.14242641201019288}, {"step": 6100, "epoch": 13, "train_loss": 0.0008753739530220628, "cos_r_negg": [0.174159437417984, 0.20411986112594604, 0.24354444444179535], "cos_innovation_negg": [0.174159437417984, 0.20411986112594604, 0.24354444444179535], "cos_apical_negg": [-0.00767859909683466, 0.028538458049297333, 0.07206092774868011], "cos_Ac_negg": [0.5765362977981567, 0.6534602642059326, 0.6382972598075867], "r_norm": [0.15607845783233643, 0.15456441044807434, 0.16035377979278564]}, {"step": 6200, "epoch": 13, "train_loss": 0.001130223972722888, "cos_r_negg": [0.17189690470695496, 0.20192931592464447, 0.24033722281455994], "cos_innovation_negg": [0.17189690470695496, 0.20192931592464447, 0.24033722281455994], "cos_apical_negg": [-0.0071361251175403595, 0.028645731508731842, 0.07237084209918976], "cos_Ac_negg": [0.5749015212059021, 0.6523905992507935, 0.6395661234855652], "r_norm": [0.15572698414325714, 0.1540389358997345, 0.16000795364379883]}, {"step": 6300, "epoch": 13, "train_loss": 0.0006541647017002106, "cos_r_negg": [0.17216068506240845, 0.1994464248418808, 0.23954151570796967], "cos_innovation_negg": [0.17216068506240845, 0.1994464248418808, 0.23954151570796967], "cos_apical_negg": [-0.0067757777869701385, 0.029019920155405998, 0.07383277267217636], "cos_Ac_negg": [0.5738915205001831, 0.6540108323097229, 0.6395518183708191], "r_norm": [0.16034150123596191, 0.15880431234836578, 0.16480137407779694]}, {"step": 6400, "epoch": 13, "train_loss": 0.0002635970595292747, "cos_r_negg": [0.1700204312801361, 0.19879797101020813, 0.23477008938789368], "cos_innovation_negg": [0.1700204312801361, 0.19879797101020813, 0.23477008938789368], "cos_apical_negg": [-0.007688488345593214, 0.02872314676642418, 0.06840697675943375], "cos_Ac_negg": [0.5727319717407227, 0.653275728225708, 0.639255166053772], "r_norm": [0.1576792597770691, 0.15623797476291656, 0.16188094019889832]}, {"step": 6500, "epoch": 13, "train_loss": 6.440469587687403e-05, "cos_r_negg": [0.1699916422367096, 0.1988181471824646, 0.2355714589357376], "cos_innovation_negg": [0.1699916422367096, 0.1988181471824646, 0.2355714589357376], "cos_apical_negg": [-0.007876312360167503, 0.02910144068300724, 0.0683816522359848], "cos_Ac_negg": [0.5729690790176392, 0.6532773971557617, 0.6397091746330261], "r_norm": [0.15950703620910645, 0.15799561142921448, 0.16392040252685547], "ldr": {"dL_sdil": -0.009079977869987488, "dL_bp": -0.01777389645576477, "ratio": 0.5108602884339689}}, {"epoch_end": 13, "step": 6566, "test_acc": 0.9732, "test_loss": 0.14537806529998779}, {"step": 6600, "epoch": 14, "train_loss": 0.0002895647776313126, "cos_r_negg": [0.16190284490585327, 0.18903881311416626, 0.2272726148366928], "cos_innovation_negg": [0.16190284490585327, 0.18903881311416626, 0.2272726148366928], "cos_apical_negg": [-0.007980301976203918, 0.028466172516345978, 0.06749342381954193], "cos_Ac_negg": [0.5734381079673767, 0.6531709432601929, 0.6392842531204224], "r_norm": [0.1596287339925766, 0.15802395343780518, 0.1638604998588562]}, {"step": 6700, "epoch": 14, "train_loss": 0.0007385698845610023, "cos_r_negg": [0.16301581263542175, 0.19068433344364166, 0.22741979360580444], "cos_innovation_negg": [0.16301581263542175, 0.19068433344364166, 0.22741979360580444], "cos_apical_negg": [-0.008966032415628433, 0.028059065341949463, 0.06547172367572784], "cos_Ac_negg": [0.5714814066886902, 0.6525571346282959, 0.6417069435119629], "r_norm": [0.15808875858783722, 0.15658102929592133, 0.16241328418254852]}, {"step": 6800, "epoch": 14, "train_loss": 0.0004385093052405864, "cos_r_negg": [0.1579444408416748, 0.18256783485412598, 0.22141847014427185], "cos_innovation_negg": [0.1579444408416748, 0.18256783485412598, 0.22141847014427185], "cos_apical_negg": [-0.007996346801519394, 0.02764541655778885, 0.06454102694988251], "cos_Ac_negg": [0.5723294615745544, 0.6518482565879822, 0.6396939754486084], "r_norm": [0.158376544713974, 0.15680678188800812, 0.16260096430778503]}, {"step": 6900, "epoch": 14, "train_loss": 0.00074908381793648, "cos_r_negg": [0.154276043176651, 0.17812806367874146, 0.21594232320785522], "cos_innovation_negg": [0.154276043176651, 0.17812806367874146, 0.21594232320785522], "cos_apical_negg": [-0.008226802572607994, 0.027054058387875557, 0.06316272914409637], "cos_Ac_negg": [0.575911283493042, 0.6519684791564941, 0.6386241316795349], "r_norm": [0.15596985816955566, 0.15434530377388, 0.15996414422988892]}, {"step": 7000, "epoch": 14, "train_loss": 0.0010558165377005935, "cos_r_negg": [0.15769124031066895, 0.18136253952980042, 0.2184697836637497], "cos_innovation_negg": [0.15769124031066895, 0.18136253952980042, 0.2184697836637497], "cos_apical_negg": [-0.0075513459742069244, 0.02734886296093464, 0.06309651583433151], "cos_Ac_negg": [0.5748131275177002, 0.6541335582733154, 0.6396275162696838], "r_norm": [0.1557645946741104, 0.15423013269901276, 0.1599709838628769], "ldr": {"dL_sdil": -0.009087041020393372, "dL_bp": -0.017992638051509857, "ratio": 0.5050421730475944}}, {"epoch_end": 14, "step": 7035, "test_acc": 0.973, "test_loss": 0.14823665733337402}], "final": {"test_acc": 0.973, "test_loss": 0.14823665733337402, "wall_s": 22.736174821853638, "cos_r_negg": [0.1556822657585144, 0.17938001453876495, 0.2172180563211441], "cos_innovation_negg": [0.1556822657585144, 0.17938001453876495, 0.2172180563211441], "cos_apical_negg": [-0.0077785924077034, 0.026926474645733833, 0.06347028911113739], "cos_Ac_negg": [0.5730034112930298, 0.6533554196357727, 0.6403895616531372]}} \ No newline at end of file diff --git a/results/pepita_tuned_v1_mnist_pepita_w256_d3_s0.json b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s0.json new file mode 100644 index 0000000..eff16bb --- /dev/null +++ b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s0.json @@ -0,0 +1 @@ +{"args": {"method": "pepita", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 60, "batch_size": 64, "max_batches": 0, "eta": 0.001, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 0, "device": "cuda", "outdir": "results", "tag": "pepita_tuned_v1_mnist_pepita_w256_d3_s0"}, "method_source": {"paper": "https://proceedings.mlr.press/v162/dellaferrera22a.html", "code": "https://github.com/GiorgiaD/PEPITA", "protocol": "ERIN two-forward-pass rule; He-uniform; ReLU; F scale 0.05"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.6518, "test_loss": 1.559471484375, "train_loss": 1.7875884771347046}, {"epoch_end": 1, "test_acc": 0.7212, "test_loss": 1.1884593200683593, "train_loss": 1.191482663154602}, {"epoch_end": 2, "test_acc": 0.7588, "test_loss": 0.9681992126464843, "train_loss": 1.1155023574829102}, {"epoch_end": 3, "test_acc": 0.7851, "test_loss": 0.8231366760253906, "train_loss": 1.0380592346191406}, {"epoch_end": 4, "test_acc": 0.8048, "test_loss": 0.7213959838867188, "train_loss": 0.9966155290603638}, {"epoch_end": 5, "test_acc": 0.8197, "test_loss": 0.6503797943115235, "train_loss": 1.0428134202957153}, {"epoch_end": 6, "test_acc": 0.8341, "test_loss": 0.5964381805419922, "train_loss": 0.8256586790084839}, {"epoch_end": 7, "test_acc": 0.8448, "test_loss": 0.554099560546875, "train_loss": 1.1859040260314941}, {"epoch_end": 8, "test_acc": 0.8521, "test_loss": 0.5201761260986328, "train_loss": 0.6205934286117554}, {"epoch_end": 9, "test_acc": 0.857, "test_loss": 0.49594663696289065, "train_loss": 0.9442326426506042}, {"epoch_end": 10, "test_acc": 0.8625, "test_loss": 0.47613604278564453, "train_loss": 1.028879165649414}, {"epoch_end": 11, "test_acc": 0.8669, "test_loss": 0.45639532775878905, "train_loss": 0.8485857248306274}, {"epoch_end": 12, "test_acc": 0.8699, "test_loss": 0.44283943786621094, "train_loss": 0.4658890664577484}, {"epoch_end": 13, "test_acc": 0.8727, "test_loss": 0.4321797439575195, "train_loss": 0.6189011931419373}, {"epoch_end": 14, "test_acc": 0.8742, "test_loss": 0.4235057464599609, "train_loss": 0.5495114326477051}, {"epoch_end": 15, "test_acc": 0.8764, "test_loss": 0.4142613540649414, "train_loss": 0.9973715543746948}, {"epoch_end": 16, "test_acc": 0.8816, "test_loss": 0.4029728500366211, "train_loss": 0.44649359583854675}, {"epoch_end": 17, "test_acc": 0.8799, "test_loss": 0.40050851440429686, "train_loss": 0.6296642422676086}, {"epoch_end": 18, "test_acc": 0.8843, "test_loss": 0.38845673675537107, "train_loss": 0.5928187966346741}, {"epoch_end": 19, "test_acc": 0.8854, "test_loss": 0.3850729965209961, "train_loss": 0.3650397062301636}, {"epoch_end": 20, "test_acc": 0.8865, "test_loss": 0.38134848327636717, "train_loss": 0.6223674416542053}, {"epoch_end": 21, "test_acc": 0.8895, "test_loss": 0.375591047668457, "train_loss": 0.3824939727783203}, {"epoch_end": 22, "test_acc": 0.8892, "test_loss": 0.37207530212402345, "train_loss": 0.35851842164993286}, {"epoch_end": 23, "test_acc": 0.8914, "test_loss": 0.3634347946166992, "train_loss": 0.9082710146903992}, {"epoch_end": 24, "test_acc": 0.8939, "test_loss": 0.36353958587646484, "train_loss": 0.6468604803085327}, {"epoch_end": 25, "test_acc": 0.8931, "test_loss": 0.36195335998535155, "train_loss": 0.4499422013759613}, {"epoch_end": 26, "test_acc": 0.8959, "test_loss": 0.35401924591064454, "train_loss": 0.4366362690925598}, {"epoch_end": 27, "test_acc": 0.8952, "test_loss": 0.3500738998413086, "train_loss": 0.5309292674064636}, {"epoch_end": 28, "test_acc": 0.8974, "test_loss": 0.3500406341552734, "train_loss": 0.6337316632270813}, {"epoch_end": 29, "test_acc": 0.8969, "test_loss": 0.34627766876220706, "train_loss": 0.808552086353302}, {"epoch_end": 30, "test_acc": 0.9003, "test_loss": 0.33843775939941406, "train_loss": 0.609764814376831}, {"epoch_end": 31, "test_acc": 0.9005, "test_loss": 0.3370256454467773, "train_loss": 0.6493722200393677}, {"epoch_end": 32, "test_acc": 0.9, "test_loss": 0.34001707458496094, "train_loss": 0.7023178339004517}, {"epoch_end": 33, "test_acc": 0.9025, "test_loss": 0.33235078277587893, "train_loss": 0.41134247183799744}, {"epoch_end": 34, "test_acc": 0.9018, "test_loss": 0.3310313827514648, "train_loss": 0.5978655219078064}, {"epoch_end": 35, "test_acc": 0.9019, "test_loss": 0.33463938598632814, "train_loss": 0.232545867562294}, {"epoch_end": 36, "test_acc": 0.9035, "test_loss": 0.3241629051208496, "train_loss": 0.632738471031189}, {"epoch_end": 37, "test_acc": 0.9035, "test_loss": 0.3257875198364258, "train_loss": 0.324849396944046}, {"epoch_end": 38, "test_acc": 0.9035, "test_loss": 0.32172699813842776, "train_loss": 0.3749930262565613}, {"epoch_end": 39, "test_acc": 0.9065, "test_loss": 0.31743964157104493, "train_loss": 0.43916159868240356}, {"epoch_end": 40, "test_acc": 0.9062, "test_loss": 0.3177797637939453, "train_loss": 0.37843817472457886}, {"epoch_end": 41, "test_acc": 0.908, "test_loss": 0.31017623138427736, "train_loss": 0.45738905668258667}, {"epoch_end": 42, "test_acc": 0.9086, "test_loss": 0.31162081680297854, "train_loss": 0.5457720160484314}, {"epoch_end": 43, "test_acc": 0.9069, "test_loss": 0.31261386947631836, "train_loss": 0.6531697511672974}, {"epoch_end": 44, "test_acc": 0.9068, "test_loss": 0.31471758880615236, "train_loss": 0.5303422808647156}, {"epoch_end": 45, "test_acc": 0.9109, "test_loss": 0.30700038681030273, "train_loss": 0.48354029655456543}, {"epoch_end": 46, "test_acc": 0.9063, "test_loss": 0.3139000663757324, "train_loss": 0.25133252143859863}, {"epoch_end": 47, "test_acc": 0.9106, "test_loss": 0.3020589294433594, "train_loss": 0.5885887742042542}, {"epoch_end": 48, "test_acc": 0.9091, "test_loss": 0.30455829391479494, "train_loss": 0.3467448353767395}, {"epoch_end": 49, "test_acc": 0.9112, "test_loss": 0.2991777130126953, "train_loss": 0.7662556767463684}, {"epoch_end": 50, "test_acc": 0.9086, "test_loss": 0.3025917900085449, "train_loss": 0.27670198678970337}, {"epoch_end": 51, "test_acc": 0.9128, "test_loss": 0.29674549102783204, "train_loss": 0.7832178473472595}, {"epoch_end": 52, "test_acc": 0.9147, "test_loss": 0.2918125511169434, "train_loss": 0.42034438252449036}, {"epoch_end": 53, "test_acc": 0.9136, "test_loss": 0.29290121002197267, "train_loss": 0.10832155495882034}, {"epoch_end": 54, "test_acc": 0.9126, "test_loss": 0.2967081443786621, "train_loss": 0.13532741367816925}, {"epoch_end": 55, "test_acc": 0.9163, "test_loss": 0.2853896705627441, "train_loss": 0.3103983998298645}, {"epoch_end": 56, "test_acc": 0.9155, "test_loss": 0.2863883743286133, "train_loss": 0.4554312527179718}, {"epoch_end": 57, "test_acc": 0.9149, "test_loss": 0.2886173545837402, "train_loss": 0.86004638671875}, {"epoch_end": 58, "test_acc": 0.9116, "test_loss": 0.29828926162719727, "train_loss": 0.6273477077484131}, {"epoch_end": 59, "test_acc": 0.9112, "test_loss": 0.2947499496459961, "train_loss": 0.5028364062309265}], "final": {"test_acc": 0.9112, "test_loss": 0.2947499496459961, "wall_s": 85.05141305923462}} \ No newline at end of file diff --git a/results/pepita_tuned_v1_mnist_pepita_w256_d3_s1.json b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s1.json new file mode 100644 index 0000000..b505709 --- /dev/null +++ b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s1.json @@ -0,0 +1 @@ +{"args": {"method": "pepita", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 60, "batch_size": 64, "max_batches": 0, "eta": 0.001, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 1, "device": "cuda", "outdir": "results", "tag": "pepita_tuned_v1_mnist_pepita_w256_d3_s1"}, "method_source": {"paper": "https://proceedings.mlr.press/v162/dellaferrera22a.html", "code": "https://github.com/GiorgiaD/PEPITA", "protocol": "ERIN two-forward-pass rule; He-uniform; ReLU; F scale 0.05"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.6576, "test_loss": 1.564298486328125, "train_loss": 1.947989583015442}, {"epoch_end": 1, "test_acc": 0.7354, "test_loss": 1.163722137451172, "train_loss": 1.1030917167663574}, {"epoch_end": 2, "test_acc": 0.7769, "test_loss": 0.9230647705078125, "train_loss": 1.266661524772644}, {"epoch_end": 3, "test_acc": 0.798, "test_loss": 0.7730442993164063, "train_loss": 0.8979678153991699}, {"epoch_end": 4, "test_acc": 0.8201, "test_loss": 0.6726371032714844, "train_loss": 1.0183836221694946}, {"epoch_end": 5, "test_acc": 0.8352, "test_loss": 0.6087283172607422, "train_loss": 0.7167853116989136}, {"epoch_end": 6, "test_acc": 0.8459, "test_loss": 0.5613793762207031, "train_loss": 0.9687842130661011}, {"epoch_end": 7, "test_acc": 0.8532, "test_loss": 0.5256933166503907, "train_loss": 0.9201305508613586}, {"epoch_end": 8, "test_acc": 0.8584, "test_loss": 0.4994579284667969, "train_loss": 0.7166524529457092}, {"epoch_end": 9, "test_acc": 0.8604, "test_loss": 0.4805103973388672, "train_loss": 0.8213995695114136}, {"epoch_end": 10, "test_acc": 0.8647, "test_loss": 0.4627833618164062, "train_loss": 1.2773122787475586}, {"epoch_end": 11, "test_acc": 0.871, "test_loss": 0.44426866455078123, "train_loss": 0.7805830240249634}, {"epoch_end": 12, "test_acc": 0.8748, "test_loss": 0.4328979797363281, "train_loss": 0.6343363523483276}, {"epoch_end": 13, "test_acc": 0.8762, "test_loss": 0.4268895965576172, "train_loss": 0.784195065498352}, {"epoch_end": 14, "test_acc": 0.8808, "test_loss": 0.4103630859375, "train_loss": 0.7097253799438477}, {"epoch_end": 15, "test_acc": 0.8836, "test_loss": 0.4051292022705078, "train_loss": 0.6262856721878052}, {"epoch_end": 16, "test_acc": 0.8844, "test_loss": 0.3944833221435547, "train_loss": 0.38616836071014404}, {"epoch_end": 17, "test_acc": 0.8804, "test_loss": 0.4045672668457031, "train_loss": 0.7168458104133606}, {"epoch_end": 18, "test_acc": 0.8888, "test_loss": 0.38385306854248047, "train_loss": 1.1206046342849731}, {"epoch_end": 19, "test_acc": 0.8854, "test_loss": 0.3848447036743164, "train_loss": 0.45324984192848206}, {"epoch_end": 20, "test_acc": 0.8852, "test_loss": 0.3811326904296875, "train_loss": 0.8075611591339111}, {"epoch_end": 21, "test_acc": 0.8876, "test_loss": 0.3744813674926758, "train_loss": 0.5532623529434204}, {"epoch_end": 22, "test_acc": 0.8953, "test_loss": 0.3603331268310547, "train_loss": 0.344644695520401}, {"epoch_end": 23, "test_acc": 0.8902, "test_loss": 0.3664552688598633, "train_loss": 1.017719030380249}, {"epoch_end": 24, "test_acc": 0.8939, "test_loss": 0.35598382720947264, "train_loss": 0.47657108306884766}, {"epoch_end": 25, "test_acc": 0.8945, "test_loss": 0.3587079376220703, "train_loss": 0.38839417695999146}, {"epoch_end": 26, "test_acc": 0.8965, "test_loss": 0.35173089294433596, "train_loss": 0.7471293807029724}, {"epoch_end": 27, "test_acc": 0.8968, "test_loss": 0.3482848815917969, "train_loss": 0.4068065583705902}, {"epoch_end": 28, "test_acc": 0.8983, "test_loss": 0.34574397888183594, "train_loss": 0.6031719446182251}, {"epoch_end": 29, "test_acc": 0.8996, "test_loss": 0.3458175537109375, "train_loss": 0.7321828007698059}, {"epoch_end": 30, "test_acc": 0.9028, "test_loss": 0.33481695709228515, "train_loss": 0.7975817322731018}, {"epoch_end": 31, "test_acc": 0.9038, "test_loss": 0.33111589584350587, "train_loss": 0.9323378801345825}, {"epoch_end": 32, "test_acc": 0.8991, "test_loss": 0.3399761764526367, "train_loss": 0.5482977032661438}, {"epoch_end": 33, "test_acc": 0.9028, "test_loss": 0.3293631561279297, "train_loss": 0.5658836364746094}, {"epoch_end": 34, "test_acc": 0.9014, "test_loss": 0.33089072647094725, "train_loss": 0.7559365630149841}, {"epoch_end": 35, "test_acc": 0.9024, "test_loss": 0.33261343688964845, "train_loss": 0.18419679999351501}, {"epoch_end": 36, "test_acc": 0.908, "test_loss": 0.3201718803405762, "train_loss": 0.43125277757644653}, {"epoch_end": 37, "test_acc": 0.9051, "test_loss": 0.32171831130981443, "train_loss": 0.333583265542984}, {"epoch_end": 38, "test_acc": 0.9072, "test_loss": 0.31539949951171875, "train_loss": 0.6099624037742615}, {"epoch_end": 39, "test_acc": 0.9037, "test_loss": 0.32065243225097656, "train_loss": 0.4864993691444397}, {"epoch_end": 40, "test_acc": 0.9084, "test_loss": 0.31198085174560547, "train_loss": 0.33445459604263306}, {"epoch_end": 41, "test_acc": 0.91, "test_loss": 0.3080244026184082, "train_loss": 0.3257055878639221}, {"epoch_end": 42, "test_acc": 0.9037, "test_loss": 0.3229133834838867, "train_loss": 0.3919094204902649}, {"epoch_end": 43, "test_acc": 0.906, "test_loss": 0.31626682891845703, "train_loss": 0.35287898778915405}, {"epoch_end": 44, "test_acc": 0.9077, "test_loss": 0.3151045967102051, "train_loss": 0.16899406909942627}, {"epoch_end": 45, "test_acc": 0.9129, "test_loss": 0.2987576126098633, "train_loss": 0.7801811695098877}, {"epoch_end": 46, "test_acc": 0.9105, "test_loss": 0.3051134475708008, "train_loss": 0.2572104036808014}, {"epoch_end": 47, "test_acc": 0.9089, "test_loss": 0.30757727508544924, "train_loss": 0.5740845203399658}, {"epoch_end": 48, "test_acc": 0.9138, "test_loss": 0.29410076599121093, "train_loss": 0.34150099754333496}, {"epoch_end": 49, "test_acc": 0.9104, "test_loss": 0.30366666870117187, "train_loss": 0.6901942491531372}, {"epoch_end": 50, "test_acc": 0.916, "test_loss": 0.2919005424499512, "train_loss": 0.5166893005371094}, {"epoch_end": 51, "test_acc": 0.9144, "test_loss": 0.29178643798828124, "train_loss": 0.8644915819168091}, {"epoch_end": 52, "test_acc": 0.9143, "test_loss": 0.293666389465332, "train_loss": 0.3433041572570801}, {"epoch_end": 53, "test_acc": 0.9128, "test_loss": 0.2942913429260254, "train_loss": 0.26649895310401917}, {"epoch_end": 54, "test_acc": 0.9107, "test_loss": 0.3001371025085449, "train_loss": 0.22972865402698517}, {"epoch_end": 55, "test_acc": 0.9157, "test_loss": 0.2882266075134277, "train_loss": 0.3866208493709564}, {"epoch_end": 56, "test_acc": 0.9172, "test_loss": 0.288204443359375, "train_loss": 0.5994341969490051}, {"epoch_end": 57, "test_acc": 0.9195, "test_loss": 0.2820064727783203, "train_loss": 0.6545388698577881}, {"epoch_end": 58, "test_acc": 0.9127, "test_loss": 0.2988055702209473, "train_loss": 0.5011070966720581}, {"epoch_end": 59, "test_acc": 0.91, "test_loss": 0.3045978668212891, "train_loss": 0.6184585690498352}], "final": {"test_acc": 0.91, "test_loss": 0.3045978668212891, "wall_s": 94.28006768226624}} \ No newline at end of file diff --git a/results/pepita_tuned_v1_mnist_pepita_w256_d3_s2.json b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s2.json new file mode 100644 index 0000000..2568588 --- /dev/null +++ b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s2.json @@ -0,0 +1 @@ +{"args": {"method": "pepita", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 60, "batch_size": 64, "max_batches": 0, "eta": 0.001, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 2, "device": "cuda", "outdir": "results", "tag": "pepita_tuned_v1_mnist_pepita_w256_d3_s2"}, "method_source": {"paper": "https://proceedings.mlr.press/v162/dellaferrera22a.html", "code": "https://github.com/GiorgiaD/PEPITA", "protocol": "ERIN two-forward-pass rule; He-uniform; ReLU; F scale 0.05"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.6813, "test_loss": 1.5333150512695313, "train_loss": 1.8331329822540283}, {"epoch_end": 1, "test_acc": 0.7448, "test_loss": 1.1189734619140626, "train_loss": 1.0263090133666992}, {"epoch_end": 2, "test_acc": 0.7738, "test_loss": 0.8981266906738281, "train_loss": 1.1194785833358765}, {"epoch_end": 3, "test_acc": 0.7928, "test_loss": 0.7678225036621094, "train_loss": 0.8481582403182983}, {"epoch_end": 4, "test_acc": 0.8114, "test_loss": 0.681844009399414, "train_loss": 0.912687361240387}, {"epoch_end": 5, "test_acc": 0.8203, "test_loss": 0.6254748016357422, "train_loss": 0.9177111983299255}, {"epoch_end": 6, "test_acc": 0.8294, "test_loss": 0.5838956573486328, "train_loss": 1.0478812456130981}, {"epoch_end": 7, "test_acc": 0.8392, "test_loss": 0.5499574340820312, "train_loss": 0.7470617890357971}, {"epoch_end": 8, "test_acc": 0.8464, "test_loss": 0.52728701171875, "train_loss": 0.8916840553283691}, {"epoch_end": 9, "test_acc": 0.8512, "test_loss": 0.5066060607910157, "train_loss": 1.0406783819198608}, {"epoch_end": 10, "test_acc": 0.8575, "test_loss": 0.48881885681152343, "train_loss": 1.5352202653884888}, {"epoch_end": 11, "test_acc": 0.8633, "test_loss": 0.4746561553955078, "train_loss": 0.6813123226165771}, {"epoch_end": 12, "test_acc": 0.8636, "test_loss": 0.46428436279296875, "train_loss": 0.6791131496429443}, {"epoch_end": 13, "test_acc": 0.863, "test_loss": 0.4584464904785156, "train_loss": 0.6615903973579407}, {"epoch_end": 14, "test_acc": 0.8703, "test_loss": 0.4403850143432617, "train_loss": 0.6212791800498962}, {"epoch_end": 15, "test_acc": 0.8702, "test_loss": 0.4320507263183594, "train_loss": 0.8495935201644897}, {"epoch_end": 16, "test_acc": 0.8747, "test_loss": 0.4268022430419922, "train_loss": 0.45476290583610535}, {"epoch_end": 17, "test_acc": 0.8725, "test_loss": 0.4227833801269531, "train_loss": 0.47910934686660767}, {"epoch_end": 18, "test_acc": 0.88, "test_loss": 0.4078524795532227, "train_loss": 1.0833230018615723}, {"epoch_end": 19, "test_acc": 0.8795, "test_loss": 0.40950572052001954, "train_loss": 0.4250057339668274}, {"epoch_end": 20, "test_acc": 0.8834, "test_loss": 0.39960771179199217, "train_loss": 0.7218810319900513}, {"epoch_end": 21, "test_acc": 0.8845, "test_loss": 0.3942512939453125, "train_loss": 0.7243726849555969}, {"epoch_end": 22, "test_acc": 0.8867, "test_loss": 0.38179217681884764, "train_loss": 0.2431107610464096}, {"epoch_end": 23, "test_acc": 0.8865, "test_loss": 0.38231748046875, "train_loss": 1.240013599395752}, {"epoch_end": 24, "test_acc": 0.8897, "test_loss": 0.3740263473510742, "train_loss": 0.6413342356681824}, {"epoch_end": 25, "test_acc": 0.886, "test_loss": 0.38184745483398436, "train_loss": 0.37310248613357544}, {"epoch_end": 26, "test_acc": 0.8906, "test_loss": 0.3745627838134766, "train_loss": 0.5281063318252563}, {"epoch_end": 27, "test_acc": 0.8918, "test_loss": 0.36877201232910156, "train_loss": 0.5490759015083313}, {"epoch_end": 28, "test_acc": 0.8881, "test_loss": 0.37047723236083985, "train_loss": 0.5695050358772278}, {"epoch_end": 29, "test_acc": 0.8909, "test_loss": 0.3669300582885742, "train_loss": 0.790963351726532}, {"epoch_end": 30, "test_acc": 0.8976, "test_loss": 0.3504394515991211, "train_loss": 0.6243597865104675}, {"epoch_end": 31, "test_acc": 0.8966, "test_loss": 0.3505338302612305, "train_loss": 0.6981348991394043}, {"epoch_end": 32, "test_acc": 0.8929, "test_loss": 0.35635341033935547, "train_loss": 0.6592001914978027}, {"epoch_end": 33, "test_acc": 0.8983, "test_loss": 0.34328984680175784, "train_loss": 0.3999021053314209}, {"epoch_end": 34, "test_acc": 0.895, "test_loss": 0.3548874496459961, "train_loss": 0.35947367548942566}, {"epoch_end": 35, "test_acc": 0.9004, "test_loss": 0.34401636657714846, "train_loss": 0.21272921562194824}, {"epoch_end": 36, "test_acc": 0.9016, "test_loss": 0.34318612365722656, "train_loss": 0.5619907379150391}, {"epoch_end": 37, "test_acc": 0.8993, "test_loss": 0.3439237838745117, "train_loss": 0.3048461973667145}, {"epoch_end": 38, "test_acc": 0.9016, "test_loss": 0.3353634826660156, "train_loss": 0.293901264667511}, {"epoch_end": 39, "test_acc": 0.9023, "test_loss": 0.33252257080078124, "train_loss": 0.27226167917251587}, {"epoch_end": 40, "test_acc": 0.9021, "test_loss": 0.3302719177246094, "train_loss": 0.3963377773761749}, {"epoch_end": 41, "test_acc": 0.9045, "test_loss": 0.3229499237060547, "train_loss": 0.5238770842552185}, {"epoch_end": 42, "test_acc": 0.9025, "test_loss": 0.3350563049316406, "train_loss": 0.5865140557289124}, {"epoch_end": 43, "test_acc": 0.8996, "test_loss": 0.33895794143676755, "train_loss": 0.38323384523391724}, {"epoch_end": 44, "test_acc": 0.9037, "test_loss": 0.32671456909179686, "train_loss": 0.34964579343795776}, {"epoch_end": 45, "test_acc": 0.9055, "test_loss": 0.32207105178833006, "train_loss": 0.5679947733879089}, {"epoch_end": 46, "test_acc": 0.9073, "test_loss": 0.32077831649780275, "train_loss": 0.30637118220329285}, {"epoch_end": 47, "test_acc": 0.9035, "test_loss": 0.3252143768310547, "train_loss": 0.3783057630062103}, {"epoch_end": 48, "test_acc": 0.9098, "test_loss": 0.31133719940185545, "train_loss": 0.40440866351127625}, {"epoch_end": 49, "test_acc": 0.903, "test_loss": 0.32272774734497073, "train_loss": 0.8575476408004761}, {"epoch_end": 50, "test_acc": 0.9088, "test_loss": 0.3135820358276367, "train_loss": 0.5738855004310608}, {"epoch_end": 51, "test_acc": 0.907, "test_loss": 0.31626825256347657, "train_loss": 0.8076278567314148}, {"epoch_end": 52, "test_acc": 0.9095, "test_loss": 0.3081289016723633, "train_loss": 0.31725478172302246}, {"epoch_end": 53, "test_acc": 0.9087, "test_loss": 0.3072570198059082, "train_loss": 0.2982637584209442}, {"epoch_end": 54, "test_acc": 0.9088, "test_loss": 0.3142477066040039, "train_loss": 0.20781180262565613}, {"epoch_end": 55, "test_acc": 0.9121, "test_loss": 0.299675984954834, "train_loss": 0.23356658220291138}, {"epoch_end": 56, "test_acc": 0.9119, "test_loss": 0.3033298980712891, "train_loss": 0.47406378388404846}, {"epoch_end": 57, "test_acc": 0.9085, "test_loss": 0.3145863327026367, "train_loss": 0.6145209670066833}, {"epoch_end": 58, "test_acc": 0.9101, "test_loss": 0.3099867012023926, "train_loss": 0.5049739480018616}, {"epoch_end": 59, "test_acc": 0.909, "test_loss": 0.30419558868408203, "train_loss": 0.7041428089141846}], "final": {"test_acc": 0.909, "test_loss": 0.30419558868408203, "wall_s": 87.32817602157593}} \ No newline at end of file diff --git a/results/pepita_tuned_v1_mnist_pepita_w256_d3_s3.json b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s3.json new file mode 100644 index 0000000..7c1bd8f --- /dev/null +++ b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s3.json @@ -0,0 +1 @@ +{"args": {"method": "pepita", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 60, "batch_size": 64, "max_batches": 0, "eta": 0.001, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 3, "device": "cuda", "outdir": "results", "tag": "pepita_tuned_v1_mnist_pepita_w256_d3_s3"}, "method_source": {"paper": "https://proceedings.mlr.press/v162/dellaferrera22a.html", "code": "https://github.com/GiorgiaD/PEPITA", "protocol": "ERIN two-forward-pass rule; He-uniform; ReLU; F scale 0.05"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.6256, "test_loss": 1.6410954345703126, "train_loss": 1.7441608905792236}, {"epoch_end": 1, "test_acc": 0.7114, "test_loss": 1.2596975708007812, "train_loss": 1.2409809827804565}, {"epoch_end": 2, "test_acc": 0.7478, "test_loss": 1.0265297912597657, "train_loss": 1.4503140449523926}, {"epoch_end": 3, "test_acc": 0.772, "test_loss": 0.8725198181152344, "train_loss": 1.2470048666000366}, {"epoch_end": 4, "test_acc": 0.7995, "test_loss": 0.7597192901611328, "train_loss": 1.1537866592407227}, {"epoch_end": 5, "test_acc": 0.8107, "test_loss": 0.6822524230957031, "train_loss": 0.7962445616722107}, {"epoch_end": 6, "test_acc": 0.8275, "test_loss": 0.6226176818847656, "train_loss": 0.6504232883453369}, {"epoch_end": 7, "test_acc": 0.8355, "test_loss": 0.5806191925048828, "train_loss": 0.6678747534751892}, {"epoch_end": 8, "test_acc": 0.8419, "test_loss": 0.5480061370849609, "train_loss": 0.9176624417304993}, {"epoch_end": 9, "test_acc": 0.8467, "test_loss": 0.5226697570800781, "train_loss": 1.0895041227340698}, {"epoch_end": 10, "test_acc": 0.8495, "test_loss": 0.5020893585205078, "train_loss": 1.1975048780441284}, {"epoch_end": 11, "test_acc": 0.8598, "test_loss": 0.4796933792114258, "train_loss": 0.7843166589736938}, {"epoch_end": 12, "test_acc": 0.8622, "test_loss": 0.4667623229980469, "train_loss": 0.6126230359077454}, {"epoch_end": 13, "test_acc": 0.8676, "test_loss": 0.4522313690185547, "train_loss": 0.8498191237449646}, {"epoch_end": 14, "test_acc": 0.8711, "test_loss": 0.44304725799560546, "train_loss": 1.0474884510040283}, {"epoch_end": 15, "test_acc": 0.8746, "test_loss": 0.4297463256835938, "train_loss": 0.9073954820632935}, {"epoch_end": 16, "test_acc": 0.8798, "test_loss": 0.41687134552001953, "train_loss": 0.5451100468635559}, {"epoch_end": 17, "test_acc": 0.8773, "test_loss": 0.41829834747314454, "train_loss": 0.7238718271255493}, {"epoch_end": 18, "test_acc": 0.8811, "test_loss": 0.40194229888916017, "train_loss": 1.028563380241394}, {"epoch_end": 19, "test_acc": 0.8858, "test_loss": 0.3939311584472656, "train_loss": 0.32413145899772644}, {"epoch_end": 20, "test_acc": 0.8853, "test_loss": 0.3909113891601563, "train_loss": 0.6150197386741638}, {"epoch_end": 21, "test_acc": 0.8902, "test_loss": 0.38175326232910156, "train_loss": 0.7616688013076782}, {"epoch_end": 22, "test_acc": 0.8889, "test_loss": 0.3776268707275391, "train_loss": 0.2595266103744507}, {"epoch_end": 23, "test_acc": 0.8925, "test_loss": 0.36983419189453126, "train_loss": 0.7195508480072021}, {"epoch_end": 24, "test_acc": 0.8924, "test_loss": 0.3710239944458008, "train_loss": 0.5820523500442505}, {"epoch_end": 25, "test_acc": 0.8928, "test_loss": 0.36393538513183593, "train_loss": 0.358381450176239}, {"epoch_end": 26, "test_acc": 0.894, "test_loss": 0.35966678314208983, "train_loss": 0.5742294192314148}, {"epoch_end": 27, "test_acc": 0.8969, "test_loss": 0.35219967193603513, "train_loss": 0.34805765748023987}, {"epoch_end": 28, "test_acc": 0.8944, "test_loss": 0.35360043182373047, "train_loss": 0.6532134413719177}, {"epoch_end": 29, "test_acc": 0.8982, "test_loss": 0.3480696517944336, "train_loss": 0.3539617657661438}, {"epoch_end": 30, "test_acc": 0.8969, "test_loss": 0.34615218048095703, "train_loss": 0.6401098966598511}, {"epoch_end": 31, "test_acc": 0.8989, "test_loss": 0.34430726165771486, "train_loss": 0.5857200622558594}, {"epoch_end": 32, "test_acc": 0.899, "test_loss": 0.34561431427001954, "train_loss": 0.6651841402053833}, {"epoch_end": 33, "test_acc": 0.9031, "test_loss": 0.33144683837890626, "train_loss": 0.49383652210235596}, {"epoch_end": 34, "test_acc": 0.902, "test_loss": 0.3300677734375, "train_loss": 0.7251609563827515}, {"epoch_end": 35, "test_acc": 0.9015, "test_loss": 0.32879483947753907, "train_loss": 0.13237632811069489}, {"epoch_end": 36, "test_acc": 0.9033, "test_loss": 0.3273560302734375, "train_loss": 0.5409269332885742}, {"epoch_end": 37, "test_acc": 0.9025, "test_loss": 0.3305997177124023, "train_loss": 0.6249890327453613}, {"epoch_end": 38, "test_acc": 0.9057, "test_loss": 0.3216881088256836, "train_loss": 0.5285733342170715}, {"epoch_end": 39, "test_acc": 0.9072, "test_loss": 0.31625783386230466, "train_loss": 0.4772343933582306}, {"epoch_end": 40, "test_acc": 0.906, "test_loss": 0.3161412628173828, "train_loss": 0.21082310378551483}, {"epoch_end": 41, "test_acc": 0.9082, "test_loss": 0.30994471740722657, "train_loss": 0.4149400293827057}, {"epoch_end": 42, "test_acc": 0.9067, "test_loss": 0.3159552993774414, "train_loss": 0.4608025550842285}, {"epoch_end": 43, "test_acc": 0.9061, "test_loss": 0.3122470634460449, "train_loss": 0.38128048181533813}, {"epoch_end": 44, "test_acc": 0.9084, "test_loss": 0.3070736297607422, "train_loss": 0.5189844965934753}, {"epoch_end": 45, "test_acc": 0.906, "test_loss": 0.3137748336791992, "train_loss": 0.7875658869743347}, {"epoch_end": 46, "test_acc": 0.9083, "test_loss": 0.30900379180908205, "train_loss": 0.2132968306541443}, {"epoch_end": 47, "test_acc": 0.911, "test_loss": 0.30109613571166993, "train_loss": 0.3097454607486725}, {"epoch_end": 48, "test_acc": 0.9085, "test_loss": 0.30484672775268556, "train_loss": 0.3194872736930847}, {"epoch_end": 49, "test_acc": 0.9103, "test_loss": 0.304660075378418, "train_loss": 0.8614277243614197}, {"epoch_end": 50, "test_acc": 0.9115, "test_loss": 0.3031661514282227, "train_loss": 0.47606706619262695}, {"epoch_end": 51, "test_acc": 0.9119, "test_loss": 0.29371373901367187, "train_loss": 0.748071014881134}, {"epoch_end": 52, "test_acc": 0.9135, "test_loss": 0.29599455184936524, "train_loss": 0.17151962220668793}, {"epoch_end": 53, "test_acc": 0.9125, "test_loss": 0.293831844329834, "train_loss": 0.1737903207540512}, {"epoch_end": 54, "test_acc": 0.9114, "test_loss": 0.29463577423095705, "train_loss": 0.1597054898738861}, {"epoch_end": 55, "test_acc": 0.9172, "test_loss": 0.28684416122436523, "train_loss": 0.4454526901245117}, {"epoch_end": 56, "test_acc": 0.9152, "test_loss": 0.2861815361022949, "train_loss": 0.439315527677536}, {"epoch_end": 57, "test_acc": 0.9152, "test_loss": 0.2846613883972168, "train_loss": 0.5872448682785034}, {"epoch_end": 58, "test_acc": 0.9165, "test_loss": 0.28581998672485354, "train_loss": 0.4633229672908783}, {"epoch_end": 59, "test_acc": 0.9145, "test_loss": 0.2839782402038574, "train_loss": 0.6298597455024719}], "final": {"test_acc": 0.9145, "test_loss": 0.2839782402038574, "wall_s": 88.9492290019989}} \ No newline at end of file diff --git a/results/pepita_tuned_v1_mnist_pepita_w256_d3_s4.json b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s4.json new file mode 100644 index 0000000..f025c51 --- /dev/null +++ b/results/pepita_tuned_v1_mnist_pepita_w256_d3_s4.json @@ -0,0 +1 @@ +{"args": {"method": "pepita", "dataset": "mnist", "depth": 3, "width": 256, "epochs": 60, "batch_size": 64, "max_batches": 0, "eta": 0.001, "momentum": 0.9, "act": "tanh", "residual": 0, "feedback_scale": 1.0, "pepita_f_scale": 0.05, "keep_prob": 0.9, "ff_threshold": 2.0, "ep_beta": 0.5, "ep_dt": 0.5, "ep_free_steps": 20, "ep_nudge_steps": 4, "canonical_preprocess": 1, "seed": 4, "device": "cuda", "outdir": "results", "tag": "pepita_tuned_v1_mnist_pepita_w256_d3_s4"}, "method_source": {"paper": "https://proceedings.mlr.press/v162/dellaferrera22a.html", "code": "https://github.com/GiorgiaD/PEPITA", "protocol": "ERIN two-forward-pass rule; He-uniform; ReLU; F scale 0.05"}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"epoch_end": 0, "test_acc": 0.6824, "test_loss": 1.4869868286132812, "train_loss": 1.7531956434249878}, {"epoch_end": 1, "test_acc": 0.7497, "test_loss": 1.1017743530273438, "train_loss": 1.0842080116271973}, {"epoch_end": 2, "test_acc": 0.7773, "test_loss": 0.90143837890625, "train_loss": 1.1783795356750488}, {"epoch_end": 3, "test_acc": 0.7913, "test_loss": 0.7864503845214844, "train_loss": 0.8495497703552246}, {"epoch_end": 4, "test_acc": 0.8062, "test_loss": 0.7060002716064453, "train_loss": 0.9830931425094604}, {"epoch_end": 5, "test_acc": 0.8164, "test_loss": 0.651647592163086, "train_loss": 0.9286741018295288}, {"epoch_end": 6, "test_acc": 0.8273, "test_loss": 0.6100898895263672, "train_loss": 0.8405406475067139}, {"epoch_end": 7, "test_acc": 0.834, "test_loss": 0.57709052734375, "train_loss": 0.6890033483505249}, {"epoch_end": 8, "test_acc": 0.8419, "test_loss": 0.551217431640625, "train_loss": 0.5836716890335083}, {"epoch_end": 9, "test_acc": 0.8453, "test_loss": 0.5291489685058594, "train_loss": 1.157970666885376}, {"epoch_end": 10, "test_acc": 0.8488, "test_loss": 0.5135103393554687, "train_loss": 1.0350550413131714}, {"epoch_end": 11, "test_acc": 0.852, "test_loss": 0.4946467437744141, "train_loss": 0.7874913811683655}, {"epoch_end": 12, "test_acc": 0.8557, "test_loss": 0.4818279006958008, "train_loss": 0.6091686487197876}, {"epoch_end": 13, "test_acc": 0.8603, "test_loss": 0.4723763275146484, "train_loss": 0.8877789974212646}, {"epoch_end": 14, "test_acc": 0.864, "test_loss": 0.45840127716064455, "train_loss": 0.9873074293136597}, {"epoch_end": 15, "test_acc": 0.8655, "test_loss": 0.4488371322631836, "train_loss": 0.7383524775505066}, {"epoch_end": 16, "test_acc": 0.8702, "test_loss": 0.4390704528808594, "train_loss": 0.5230501890182495}, {"epoch_end": 17, "test_acc": 0.8679, "test_loss": 0.43823190612792967, "train_loss": 0.8518657088279724}, {"epoch_end": 18, "test_acc": 0.8773, "test_loss": 0.4201535217285156, "train_loss": 0.8043522834777832}, {"epoch_end": 19, "test_acc": 0.8756, "test_loss": 0.41898763732910155, "train_loss": 0.533988356590271}, {"epoch_end": 20, "test_acc": 0.8778, "test_loss": 0.4107715255737305, "train_loss": 0.8357260227203369}, {"epoch_end": 21, "test_acc": 0.8801, "test_loss": 0.40354792022705077, "train_loss": 0.6339592933654785}, {"epoch_end": 22, "test_acc": 0.8817, "test_loss": 0.3980406677246094, "train_loss": 0.2728242874145508}, {"epoch_end": 23, "test_acc": 0.8793, "test_loss": 0.395425, "train_loss": 0.9432435035705566}, {"epoch_end": 24, "test_acc": 0.8847, "test_loss": 0.38849000549316404, "train_loss": 0.7034302949905396}, {"epoch_end": 25, "test_acc": 0.881, "test_loss": 0.3888999237060547, "train_loss": 0.4238361120223999}, {"epoch_end": 26, "test_acc": 0.8877, "test_loss": 0.37956517791748046, "train_loss": 0.6690238118171692}, {"epoch_end": 27, "test_acc": 0.8867, "test_loss": 0.3770872619628906, "train_loss": 0.44388896226882935}, {"epoch_end": 28, "test_acc": 0.89, "test_loss": 0.37217574920654295, "train_loss": 0.5289719104766846}, {"epoch_end": 29, "test_acc": 0.8903, "test_loss": 0.371275715637207, "train_loss": 0.8053065538406372}, {"epoch_end": 30, "test_acc": 0.8925, "test_loss": 0.36220329132080076, "train_loss": 0.7304819226264954}, {"epoch_end": 31, "test_acc": 0.8925, "test_loss": 0.3611178421020508, "train_loss": 0.8202845454216003}, {"epoch_end": 32, "test_acc": 0.8897, "test_loss": 0.36374439544677734, "train_loss": 0.6860420107841492}, {"epoch_end": 33, "test_acc": 0.8962, "test_loss": 0.3509176956176758, "train_loss": 0.4144265353679657}, {"epoch_end": 34, "test_acc": 0.8962, "test_loss": 0.3487733947753906, "train_loss": 0.6269606947898865}, {"epoch_end": 35, "test_acc": 0.8954, "test_loss": 0.35007349700927737, "train_loss": 0.17079536616802216}, {"epoch_end": 36, "test_acc": 0.9004, "test_loss": 0.34047691040039063, "train_loss": 0.6055735349655151}, {"epoch_end": 37, "test_acc": 0.897, "test_loss": 0.35004290466308596, "train_loss": 0.3641662001609802}, {"epoch_end": 38, "test_acc": 0.8962, "test_loss": 0.3461878784179688, "train_loss": 0.5664220452308655}, {"epoch_end": 39, "test_acc": 0.8998, "test_loss": 0.3385594665527344, "train_loss": 0.46959754824638367}, {"epoch_end": 40, "test_acc": 0.9018, "test_loss": 0.33266646270751954, "train_loss": 0.27531468868255615}, {"epoch_end": 41, "test_acc": 0.903, "test_loss": 0.3260057815551758, "train_loss": 0.3198343515396118}, {"epoch_end": 42, "test_acc": 0.9031, "test_loss": 0.32808352966308596, "train_loss": 0.46574699878692627}, {"epoch_end": 43, "test_acc": 0.9039, "test_loss": 0.32679202423095705, "train_loss": 0.5436965227127075}, {"epoch_end": 44, "test_acc": 0.9018, "test_loss": 0.3298073455810547, "train_loss": 0.5583456754684448}, {"epoch_end": 45, "test_acc": 0.9037, "test_loss": 0.32559867095947265, "train_loss": 0.5754497647285461}, {"epoch_end": 46, "test_acc": 0.904, "test_loss": 0.32708381958007815, "train_loss": 0.39357006549835205}, {"epoch_end": 47, "test_acc": 0.9058, "test_loss": 0.31742208404541017, "train_loss": 0.6287240982055664}, {"epoch_end": 48, "test_acc": 0.9025, "test_loss": 0.3280700668334961, "train_loss": 0.29452720284461975}, {"epoch_end": 49, "test_acc": 0.9085, "test_loss": 0.30970793533325197, "train_loss": 0.5569313764572144}, {"epoch_end": 50, "test_acc": 0.906, "test_loss": 0.31922342834472656, "train_loss": 0.4181303381919861}, {"epoch_end": 51, "test_acc": 0.9093, "test_loss": 0.3106561851501465, "train_loss": 0.8264469504356384}, {"epoch_end": 52, "test_acc": 0.9102, "test_loss": 0.30346087493896484, "train_loss": 0.2986334562301636}, {"epoch_end": 53, "test_acc": 0.9094, "test_loss": 0.3025904792785645, "train_loss": 0.3138487637042999}, {"epoch_end": 54, "test_acc": 0.9075, "test_loss": 0.31118170242309573, "train_loss": 0.08304808288812637}, {"epoch_end": 55, "test_acc": 0.9105, "test_loss": 0.2993781723022461, "train_loss": 0.5201870799064636}, {"epoch_end": 56, "test_acc": 0.912, "test_loss": 0.30018479919433594, "train_loss": 0.4827938973903656}, {"epoch_end": 57, "test_acc": 0.9112, "test_loss": 0.30066899490356447, "train_loss": 0.7296208739280701}, {"epoch_end": 58, "test_acc": 0.9101, "test_loss": 0.307449365234375, "train_loss": 0.3292308747768402}, {"epoch_end": 59, "test_acc": 0.91, "test_loss": 0.30647373809814454, "train_loss": 0.29832029342651367}], "final": {"test_acc": 0.91, "test_loss": 0.30647373809814454, "wall_s": 90.56290221214294}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d30_s0.json b/results/scale_v3_cifar10_bp_w64_d30_s0.json new file mode 100644 index 0000000..44d5bee --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d30_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d30_s0", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.415728807449341}, {"epoch_end": 0, "step": 391, "test_acc": 0.3648, "test_loss": 1.8013721069335937}, {"step": 500, "epoch": 1, "train_loss": 1.6938464641571045}, {"epoch_end": 1, "step": 782, "test_acc": 0.3924, "test_loss": 1.7378808959960939}, {"step": 1000, "epoch": 2, "train_loss": 1.7924134731292725}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4074, "test_loss": 1.7012336791992186}, {"step": 1500, "epoch": 3, "train_loss": 1.6040619611740112}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4164, "test_loss": 1.68049423828125}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4131, "test_loss": 1.6608419555664062}], "final": {"test_acc": 0.4131, "test_loss": 1.6608419555664062, "wall_s": 29.739592790603638}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d30_s1.json b/results/scale_v3_cifar10_bp_w64_d30_s1.json new file mode 100644 index 0000000..8022c02 --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d30_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d30_s1", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.638410806655884}, {"epoch_end": 0, "step": 391, "test_acc": 0.364, "test_loss": 1.8112511840820313}, {"step": 500, "epoch": 1, "train_loss": 1.6501820087432861}, {"epoch_end": 1, "step": 782, "test_acc": 0.3952, "test_loss": 1.7362552856445312}, {"step": 1000, "epoch": 2, "train_loss": 1.7208136320114136}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4051, "test_loss": 1.7023025390625}, {"step": 1500, "epoch": 3, "train_loss": 1.63710618019104}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4021, "test_loss": 1.7008232299804686}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4174, "test_loss": 1.6535031127929687}], "final": {"test_acc": 0.4174, "test_loss": 1.6535031127929687, "wall_s": 30.375305891036987}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d30_s2.json b/results/scale_v3_cifar10_bp_w64_d30_s2.json new file mode 100644 index 0000000..7bb9d1d --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d30_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d30_s2", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.7673251628875732}, {"epoch_end": 0, "step": 391, "test_acc": 0.361, "test_loss": 1.8110815307617187}, {"step": 500, "epoch": 1, "train_loss": 1.7386435270309448}, {"epoch_end": 1, "step": 782, "test_acc": 0.3907, "test_loss": 1.76063857421875}, {"step": 1000, "epoch": 2, "train_loss": 1.689368486404419}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4017, "test_loss": 1.7096170776367188}, {"step": 1500, "epoch": 3, "train_loss": 1.5534716844558716}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4079, "test_loss": 1.6882278442382812}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4252, "test_loss": 1.6541592407226562}], "final": {"test_acc": 0.4252, "test_loss": 1.6541592407226562, "wall_s": 28.97341775894165}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d30_s3.json b/results/scale_v3_cifar10_bp_w64_d30_s3.json new file mode 100644 index 0000000..ae57bd0 --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d30_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d30_s3", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.614835262298584}, {"epoch_end": 0, "step": 391, "test_acc": 0.3633, "test_loss": 1.81833056640625}, {"step": 500, "epoch": 1, "train_loss": 1.6631642580032349}, {"epoch_end": 1, "step": 782, "test_acc": 0.3902, "test_loss": 1.7558072021484374}, {"step": 1000, "epoch": 2, "train_loss": 1.8065412044525146}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3968, "test_loss": 1.7200898559570312}, {"step": 1500, "epoch": 3, "train_loss": 1.6183756589889526}, {"epoch_end": 3, "step": 1564, "test_acc": 0.409, "test_loss": 1.6976827880859375}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4122, "test_loss": 1.6841212036132813}], "final": {"test_acc": 0.4122, "test_loss": 1.6841212036132813, "wall_s": 28.322287797927856}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d30_s4.json b/results/scale_v3_cifar10_bp_w64_d30_s4.json new file mode 100644 index 0000000..a1582a1 --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d30_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d30_s4", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6514732837677}, {"epoch_end": 0, "step": 391, "test_acc": 0.3689, "test_loss": 1.8017806396484375}, {"step": 500, "epoch": 1, "train_loss": 1.7060035467147827}, {"epoch_end": 1, "step": 782, "test_acc": 0.3952, "test_loss": 1.7331631591796874}, {"step": 1000, "epoch": 2, "train_loss": 1.7220714092254639}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4018, "test_loss": 1.7134476806640624}, {"step": 1500, "epoch": 3, "train_loss": 1.674729347229004}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4103, "test_loss": 1.6911321411132811}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4275, "test_loss": 1.647104541015625}], "final": {"test_acc": 0.4275, "test_loss": 1.647104541015625, "wall_s": 31.765539407730103}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d60_s0.json b/results/scale_v3_cifar10_bp_w64_d60_s0.json new file mode 100644 index 0000000..a54954f --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d60_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d60_s0", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6154472827911377}, {"epoch_end": 0, "step": 391, "test_acc": 0.3756, "test_loss": 1.7967720581054687}, {"step": 500, "epoch": 1, "train_loss": 1.764350414276123}, {"epoch_end": 1, "step": 782, "test_acc": 0.3947, "test_loss": 1.7364661865234374}, {"step": 1000, "epoch": 2, "train_loss": 1.71304452419281}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4056, "test_loss": 1.7077377685546875}, {"step": 1500, "epoch": 3, "train_loss": 1.5759260654449463}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4126, "test_loss": 1.6889246704101561}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4293, "test_loss": 1.6495174194335938}], "final": {"test_acc": 0.4293, "test_loss": 1.6495174194335938, "wall_s": 58.37085485458374}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d60_s1.json b/results/scale_v3_cifar10_bp_w64_d60_s1.json new file mode 100644 index 0000000..bbfb4fc --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d60_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d60_s1", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4734690189361572}, {"epoch_end": 0, "step": 391, "test_acc": 0.3725, "test_loss": 1.792130712890625}, {"step": 500, "epoch": 1, "train_loss": 1.6439776420593262}, {"epoch_end": 1, "step": 782, "test_acc": 0.3979, "test_loss": 1.7440376586914061}, {"step": 1000, "epoch": 2, "train_loss": 1.7593799829483032}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4051, "test_loss": 1.7121396484375}, {"step": 1500, "epoch": 3, "train_loss": 1.6305463314056396}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4071, "test_loss": 1.7027875366210938}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4159, "test_loss": 1.675144091796875}], "final": {"test_acc": 0.4159, "test_loss": 1.675144091796875, "wall_s": 59.24861788749695}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d60_s2.json b/results/scale_v3_cifar10_bp_w64_d60_s2.json new file mode 100644 index 0000000..14a5cc4 --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d60_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d60_s2", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.45672869682312}, {"epoch_end": 0, "step": 391, "test_acc": 0.3652, "test_loss": 1.7949141723632813}, {"step": 500, "epoch": 1, "train_loss": 1.7190427780151367}, {"epoch_end": 1, "step": 782, "test_acc": 0.3978, "test_loss": 1.7412082763671874}, {"step": 1000, "epoch": 2, "train_loss": 1.7992725372314453}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4006, "test_loss": 1.7093327026367187}, {"step": 1500, "epoch": 3, "train_loss": 1.6400798559188843}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4091, "test_loss": 1.6874370849609375}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4138, "test_loss": 1.663275927734375}], "final": {"test_acc": 0.4138, "test_loss": 1.663275927734375, "wall_s": 59.22583985328674}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d60_s3.json b/results/scale_v3_cifar10_bp_w64_d60_s3.json new file mode 100644 index 0000000..a11ce4b --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d60_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d60_s3", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4164717197418213}, {"epoch_end": 0, "step": 391, "test_acc": 0.3643, "test_loss": 1.7990243774414063}, {"step": 500, "epoch": 1, "train_loss": 1.651111364364624}, {"epoch_end": 1, "step": 782, "test_acc": 0.3959, "test_loss": 1.7507692626953124}, {"step": 1000, "epoch": 2, "train_loss": 1.7603168487548828}, {"epoch_end": 2, "step": 1173, "test_acc": 0.403, "test_loss": 1.7118653442382812}, {"step": 1500, "epoch": 3, "train_loss": 1.7584331035614014}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4114, "test_loss": 1.6929181884765625}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4138, "test_loss": 1.6663089965820312}], "final": {"test_acc": 0.4138, "test_loss": 1.6663089965820312, "wall_s": 56.848419189453125}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_bp_w64_d60_s4.json b/results/scale_v3_cifar10_bp_w64_d60_s4.json new file mode 100644 index 0000000..af2ddd6 --- /dev/null +++ b/results/scale_v3_cifar10_bp_w64_d60_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "bp", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_bp_w64_d60_s4", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6918694972991943}, {"epoch_end": 0, "step": 391, "test_acc": 0.3628, "test_loss": 1.8112965454101562}, {"step": 500, "epoch": 1, "train_loss": 1.6508883237838745}, {"epoch_end": 1, "step": 782, "test_acc": 0.392, "test_loss": 1.744244287109375}, {"step": 1000, "epoch": 2, "train_loss": 1.698557734489441}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4044, "test_loss": 1.7174929931640626}, {"step": 1500, "epoch": 3, "train_loss": 1.5780103206634521}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4069, "test_loss": 1.6994436767578125}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4084, "test_loss": 1.6839714233398437}], "final": {"test_acc": 0.4084, "test_loss": 1.6839714233398437, "wall_s": 58.33043193817139}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d30_s0.json b/results/scale_v3_cifar10_dfa_w64_d30_s0.json new file mode 100644 index 0000000..ffe4c41 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d30_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d30_s0", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4157285690307617, "cos_r_negg": [0.04130179435014725, -0.05167870968580246, 0.029879383742809296, 0.03635120019316673, -0.00517943175509572, -0.0037146983668208122, 0.014719751663506031, -0.012813427485525608, 0.08911487460136414, -0.025006676092743874, -0.043925944715738297, 0.004191750194877386, 0.0028503923676908016, 0.03278622776269913, -0.023855693638324738, 0.03571062535047531, -0.015357001684606075, -2.7436763048171997e-05, 0.010302920825779438, 0.02555227465927601, 0.040136873722076416, -0.010950050316751003, 0.03490540385246277, -0.029108267277479172, -0.04761591553688049, -0.03802322596311569, 0.015627088025212288, 0.0047096796333789825, 0.055373527109622955, 0.043867871165275574], "cos_innovation_negg": [0.04130179435014725, -0.05167870968580246, 0.029879383742809296, 0.03635120019316673, -0.00517943175509572, -0.0037146983668208122, 0.014719751663506031, -0.012813427485525608, 0.08911487460136414, -0.025006676092743874, -0.043925944715738297, 0.004191750194877386, 0.0028503923676908016, 0.03278622776269913, -0.023855693638324738, 0.03571062535047531, -0.015357001684606075, -2.7436763048171997e-05, 0.010302920825779438, 0.02555227465927601, 0.040136873722076416, -0.010950050316751003, 0.03490540385246277, -0.029108267277479172, -0.04761591553688049, -0.03802322596311569, 0.015627088025212288, 0.0047096796333789825, 0.055373527109622955, 0.043867871165275574], "cos_apical_negg": [0.04130179435014725, -0.05167870968580246, 0.029879383742809296, 0.03635120019316673, -0.00517943175509572, -0.0037146983668208122, 0.014719751663506031, -0.012813427485525608, 0.08911487460136414, -0.025006676092743874, -0.043925944715738297, 0.004191750194877386, 0.0028503923676908016, 0.03278622776269913, -0.023855693638324738, 0.03571062535047531, -0.015357001684606075, -2.7436763048171997e-05, 0.010302920825779438, 0.02555227465927601, 0.040136873722076416, -0.010950050316751003, 0.03490540385246277, -0.029108267277479172, -0.04761591553688049, -0.03802322596311569, 0.015627088025212288, 0.0047096796333789825, 0.055373527109622955, 0.043867871165275574], "cos_Ac_negg": [0.04130179435014725, -0.05167870968580246, 0.029879383742809296, 0.03635120019316673, -0.00517943175509572, -0.0037146983668208122, 0.014719751663506031, -0.012813427485525608, 0.08911487460136414, -0.025006676092743874, -0.043925944715738297, 0.004191750194877386, 0.0028503923676908016, 0.03278622776269913, -0.023855693638324738, 0.03571062535047531, -0.015357001684606075, -2.7436763048171997e-05, 0.010302920825779438, 0.02555227465927601, 0.040136873722076416, -0.010950050316751003, 0.03490540385246277, -0.029108267277479172, -0.04761591553688049, -0.03802322596311569, 0.015627088025212288, 0.0047096796333789825, 0.055373527109622955, 0.043867871165275574], "r_norm": [2.6118388175964355, 2.5107076168060303, 2.381376266479492, 2.5533318519592285, 2.4093616008758545, 2.4340720176696777, 2.5209712982177734, 2.47703218460083, 2.3975400924682617, 2.395562171936035, 2.4695844650268555, 2.3991799354553223, 2.5008273124694824, 2.5226407051086426, 2.46388578414917, 2.44757080078125, 2.4495849609375, 2.4044461250305176, 2.402073860168457, 2.440375566482544, 2.5851316452026367, 2.459933280944824, 2.5742692947387695, 2.475496292114258, 2.421234607696533, 2.4259753227233887, 2.365795135498047, 2.4156174659729004, 2.435579299926758, 2.5802385807037354]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3439, "test_loss": 1.8688878173828125}, {"step": 500, "epoch": 1, "train_loss": 1.8185863494873047, "cos_r_negg": [0.5021451711654663, -0.024272777140140533, -0.010427339933812618, 0.028790874406695366, -0.007071430329233408, 0.041475072503089905, -0.06685444712638855, -0.0792425200343132, 0.03061535768210888, 0.013030911795794964, -0.013616947457194328, -0.005817234516143799, -0.05089216306805611, -0.022574737668037415, 0.00764061463996768, 0.008589653298258781, -0.08776582777500153, 0.027161289006471634, 0.008954466320574284, -0.01848287507891655, -0.03218787536025047, -0.013208515010774136, 0.016982974484562874, 0.11147912591695786, -0.0019691158086061478, 0.06348760426044464, 0.05869927257299423, 0.008566780015826225, 0.0175030454993248, -0.015016330406069756], "cos_innovation_negg": [0.5021451711654663, -0.024272777140140533, -0.010427339933812618, 0.028790874406695366, -0.007071430329233408, 0.041475072503089905, -0.06685444712638855, -0.0792425200343132, 0.03061535768210888, 0.013030911795794964, -0.013616947457194328, -0.005817234516143799, -0.05089216306805611, -0.022574737668037415, 0.00764061463996768, 0.008589653298258781, -0.08776582777500153, 0.027161289006471634, 0.008954466320574284, -0.01848287507891655, -0.03218787536025047, -0.013208515010774136, 0.016982974484562874, 0.11147912591695786, -0.0019691158086061478, 0.06348760426044464, 0.05869927257299423, 0.008566780015826225, 0.0175030454993248, -0.015016330406069756], "cos_apical_negg": [0.5021451711654663, -0.024272777140140533, -0.010427339933812618, 0.028790874406695366, -0.007071430329233408, 0.041475072503089905, -0.06685444712638855, -0.0792425200343132, 0.03061535768210888, 0.013030911795794964, -0.013616947457194328, -0.005817234516143799, -0.05089216306805611, -0.022574737668037415, 0.00764061463996768, 0.008589653298258781, -0.08776582777500153, 0.027161289006471634, 0.008954466320574284, -0.01848287507891655, -0.03218787536025047, -0.013208515010774136, 0.016982974484562874, 0.11147912591695786, -0.0019691158086061478, 0.06348760426044464, 0.05869927257299423, 0.008566780015826225, 0.0175030454993248, -0.015016330406069756], "cos_Ac_negg": [0.5021451711654663, -0.024272777140140533, -0.010427339933812618, 0.028790874406695366, -0.007071430329233408, 0.041475072503089905, -0.06685444712638855, -0.0792425200343132, 0.03061535768210888, 0.013030911795794964, -0.013616947457194328, -0.005817234516143799, -0.05089216306805611, -0.022574737668037415, 0.00764061463996768, 0.008589653298258781, -0.08776582777500153, 0.027161289006471634, 0.008954466320574284, -0.01848287507891655, -0.03218787536025047, -0.013208515010774136, 0.016982974484562874, 0.11147912591695786, -0.0019691158086061478, 0.06348760426044464, 0.05869927257299423, 0.008566780015826225, 0.0175030454993248, -0.015016330406069756], "r_norm": [2.2987172603607178, 2.2241992950439453, 2.116129159927368, 2.276242733001709, 2.134511947631836, 2.147556781768799, 2.2263612747192383, 2.191457509994507, 2.1340630054473877, 2.1296725273132324, 2.2165956497192383, 2.130927324295044, 2.2357640266418457, 2.2310521602630615, 2.1970715522766113, 2.1958048343658447, 2.178600788116455, 2.134237766265869, 2.1169447898864746, 2.1933095455169678, 2.276766777038574, 2.211942195892334, 2.2697978019714355, 2.16856050491333, 2.185072183609009, 2.142336368560791, 2.078704357147217, 2.170818567276001, 2.163302421569824, 2.2897000312805176]}, {"epoch_end": 1, "step": 782, "test_acc": 0.357, "test_loss": 1.8430688598632812}, {"step": 1000, "epoch": 2, "train_loss": 1.8754260540008545, "cos_r_negg": [0.5487571954727173, 0.007248611655086279, -0.008624782785773277, -0.0057631745003163815, -0.031145749613642693, 0.052426233887672424, -0.044822391122579575, -0.07710064947605133, 0.020750969648361206, 0.026115912944078445, -0.02820703759789467, 0.016865402460098267, 0.0038375824224203825, -0.024961546063423157, 0.01999833434820175, -0.02775469422340393, -0.06761563569307327, 0.04648683965206146, 0.01960933953523636, -0.0023083328269422054, -0.06000566482543945, -0.05465877056121826, 0.016128091141581535, 0.07718849182128906, 0.01232006773352623, 0.06577295064926147, 0.07419411838054657, -0.017742210999131203, 0.01880396343767643, 0.05452614277601242], "cos_innovation_negg": [0.5487571954727173, 0.007248611655086279, -0.008624782785773277, -0.0057631745003163815, -0.031145749613642693, 0.052426233887672424, -0.044822391122579575, -0.07710064947605133, 0.020750969648361206, 0.026115912944078445, -0.02820703759789467, 0.016865402460098267, 0.0038375824224203825, -0.024961546063423157, 0.01999833434820175, -0.02775469422340393, -0.06761563569307327, 0.04648683965206146, 0.01960933953523636, -0.0023083328269422054, -0.06000566482543945, -0.05465877056121826, 0.016128091141581535, 0.07718849182128906, 0.01232006773352623, 0.06577295064926147, 0.07419411838054657, -0.017742210999131203, 0.01880396343767643, 0.05452614277601242], "cos_apical_negg": [0.5487571954727173, 0.007248611655086279, -0.008624782785773277, -0.0057631745003163815, -0.031145749613642693, 0.052426233887672424, -0.044822391122579575, -0.07710064947605133, 0.020750969648361206, 0.026115912944078445, -0.02820703759789467, 0.016865402460098267, 0.0038375824224203825, -0.024961546063423157, 0.01999833434820175, -0.02775469422340393, -0.06761563569307327, 0.04648683965206146, 0.01960933953523636, -0.0023083328269422054, -0.06000566482543945, -0.05465877056121826, 0.016128091141581535, 0.07718849182128906, 0.01232006773352623, 0.06577295064926147, 0.07419411838054657, -0.017742210999131203, 0.01880396343767643, 0.05452614277601242], "cos_Ac_negg": [0.5487571954727173, 0.007248611655086279, -0.008624782785773277, -0.0057631745003163815, -0.031145749613642693, 0.052426233887672424, -0.044822391122579575, -0.07710064947605133, 0.020750969648361206, 0.026115912944078445, -0.02820703759789467, 0.016865402460098267, 0.0038375824224203825, -0.024961546063423157, 0.01999833434820175, -0.02775469422340393, -0.06761563569307327, 0.04648683965206146, 0.01960933953523636, -0.0023083328269422054, -0.06000566482543945, -0.05465877056121826, 0.016128091141581535, 0.07718849182128906, 0.01232006773352623, 0.06577295064926147, 0.07419411838054657, -0.017742210999131203, 0.01880396343767643, 0.05452614277601242], "r_norm": [2.3114936351776123, 2.2366087436676025, 2.122636079788208, 2.288330316543579, 2.1441550254821777, 2.16719388961792, 2.238264560699463, 2.2185521125793457, 2.143141984939575, 2.1465981006622314, 2.2425267696380615, 2.140319347381592, 2.2467145919799805, 2.2544517517089844, 2.2014012336730957, 2.210102081298828, 2.202552318572998, 2.152041435241699, 2.1359610557556152, 2.2039899826049805, 2.271368980407715, 2.221750020980835, 2.2761611938476562, 2.179783344268799, 2.194851875305176, 2.147956609725952, 2.0958988666534424, 2.188201904296875, 2.171701192855835, 2.3094558715820312]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3618, "test_loss": 1.8217102172851563}, {"step": 1500, "epoch": 3, "train_loss": 1.7128980159759521, "cos_r_negg": [0.5362956523895264, 0.011473788879811764, -0.011900755576789379, 0.02257465198636055, 0.021306442096829414, 0.04985255002975464, -0.02962353453040123, -0.019608264788985252, 0.03569295257329941, 0.07186584174633026, -0.007914965972304344, 0.008661587722599506, -0.053245604038238525, 0.08209188282489777, 0.05064284801483154, 0.0038046101108193398, 0.021014947444200516, 0.08605606853961945, 0.014818879775702953, 0.03713211417198181, -0.0714733898639679, 0.02223656326532364, 0.05603468045592308, 0.06525898724794388, 0.021623045206069946, 0.058812737464904785, 0.08270993828773499, 0.028872953727841377, 0.009778222069144249, 0.0332469716668129], "cos_innovation_negg": [0.5362956523895264, 0.011473788879811764, -0.011900755576789379, 0.02257465198636055, 0.021306442096829414, 0.04985255002975464, -0.02962353453040123, -0.019608264788985252, 0.03569295257329941, 0.07186584174633026, -0.007914965972304344, 0.008661587722599506, -0.053245604038238525, 0.08209188282489777, 0.05064284801483154, 0.0038046101108193398, 0.021014947444200516, 0.08605606853961945, 0.014818879775702953, 0.03713211417198181, -0.0714733898639679, 0.02223656326532364, 0.05603468045592308, 0.06525898724794388, 0.021623045206069946, 0.058812737464904785, 0.08270993828773499, 0.028872953727841377, 0.009778222069144249, 0.0332469716668129], "cos_apical_negg": [0.5362956523895264, 0.011473788879811764, -0.011900755576789379, 0.02257465198636055, 0.021306442096829414, 0.04985255002975464, -0.02962353453040123, -0.019608264788985252, 0.03569295257329941, 0.07186584174633026, -0.007914965972304344, 0.008661587722599506, -0.053245604038238525, 0.08209188282489777, 0.05064284801483154, 0.0038046101108193398, 0.021014947444200516, 0.08605606853961945, 0.014818879775702953, 0.03713211417198181, -0.0714733898639679, 0.02223656326532364, 0.05603468045592308, 0.06525898724794388, 0.021623045206069946, 0.058812737464904785, 0.08270993828773499, 0.028872953727841377, 0.009778222069144249, 0.0332469716668129], "cos_Ac_negg": [0.5362956523895264, 0.011473788879811764, -0.011900755576789379, 0.02257465198636055, 0.021306442096829414, 0.04985255002975464, -0.02962353453040123, -0.019608264788985252, 0.03569295257329941, 0.07186584174633026, -0.007914965972304344, 0.008661587722599506, -0.053245604038238525, 0.08209188282489777, 0.05064284801483154, 0.0038046101108193398, 0.021014947444200516, 0.08605606853961945, 0.014818879775702953, 0.03713211417198181, -0.0714733898639679, 0.02223656326532364, 0.05603468045592308, 0.06525898724794388, 0.021623045206069946, 0.058812737464904785, 0.08270993828773499, 0.028872953727841377, 0.009778222069144249, 0.0332469716668129], "r_norm": [2.2674660682678223, 2.2026212215423584, 2.090864658355713, 2.2454700469970703, 2.1055290699005127, 2.118560791015625, 2.190725326538086, 2.1559906005859375, 2.1015584468841553, 2.0963563919067383, 2.183671712875366, 2.101579427719116, 2.1998825073242188, 2.208488702774048, 2.1671369075775146, 2.1577255725860596, 2.1452183723449707, 2.109074831008911, 2.090406656265259, 2.1647396087646484, 2.2389886379241943, 2.178550958633423, 2.2406816482543945, 2.1366395950317383, 2.152134895324707, 2.110382556915283, 2.0520753860473633, 2.142508029937744, 2.135183334350586, 2.253416061401367]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3724, "test_loss": 1.81636044921875}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3731, "test_loss": 1.8001628662109375}], "final": {"test_acc": 0.3731, "test_loss": 1.8001628662109375, "wall_s": 22.79020619392395, "cos_r_negg": [0.6099951267242432, 0.041578128933906555, -0.006816086359322071, 0.03268715739250183, 0.020214516669511795, 0.04716081917285919, -0.008767107501626015, -0.040753573179244995, 0.05464443564414978, 0.09139872342348099, -0.003479434410110116, 0.03690046817064285, -0.019361397251486778, 0.0345904566347599, 0.018553491681814194, 0.07845322787761688, -0.011157560162246227, 0.0830966979265213, 0.08898749947547913, 0.009537972509860992, -0.009123572148382664, -0.004601208493113518, 0.028817608952522278, 0.07400146126747131, 0.0473867766559124, 0.11561077833175659, 0.06366584450006485, -0.02335822954773903, 0.06951933354139328, 0.019080501049757004], "cos_innovation_negg": [0.6099951267242432, 0.041578128933906555, -0.006816086359322071, 0.03268715739250183, 0.020214516669511795, 0.04716081917285919, -0.008767107501626015, -0.040753573179244995, 0.05464443564414978, 0.09139872342348099, -0.003479434410110116, 0.03690046817064285, -0.019361397251486778, 0.0345904566347599, 0.018553491681814194, 0.07845322787761688, -0.011157560162246227, 0.0830966979265213, 0.08898749947547913, 0.009537972509860992, -0.009123572148382664, -0.004601208493113518, 0.028817608952522278, 0.07400146126747131, 0.0473867766559124, 0.11561077833175659, 0.06366584450006485, -0.02335822954773903, 0.06951933354139328, 0.019080501049757004], "cos_apical_negg": [0.6099951267242432, 0.041578128933906555, -0.006816086359322071, 0.03268715739250183, 0.020214516669511795, 0.04716081917285919, -0.008767107501626015, -0.040753573179244995, 0.05464443564414978, 0.09139872342348099, -0.003479434410110116, 0.03690046817064285, -0.019361397251486778, 0.0345904566347599, 0.018553491681814194, 0.07845322787761688, -0.011157560162246227, 0.0830966979265213, 0.08898749947547913, 0.009537972509860992, -0.009123572148382664, -0.004601208493113518, 0.028817608952522278, 0.07400146126747131, 0.0473867766559124, 0.11561077833175659, 0.06366584450006485, -0.02335822954773903, 0.06951933354139328, 0.019080501049757004], "cos_Ac_negg": [0.6099951267242432, 0.041578128933906555, -0.006816086359322071, 0.03268715739250183, 0.020214516669511795, 0.04716081917285919, -0.008767107501626015, -0.040753573179244995, 0.05464443564414978, 0.09139872342348099, -0.003479434410110116, 0.03690046817064285, -0.019361397251486778, 0.0345904566347599, 0.018553491681814194, 0.07845322787761688, -0.011157560162246227, 0.0830966979265213, 0.08898749947547913, 0.009537972509860992, -0.009123572148382664, -0.004601208493113518, 0.028817608952522278, 0.07400146126747131, 0.0473867766559124, 0.11561077833175659, 0.06366584450006485, -0.02335822954773903, 0.06951933354139328, 0.019080501049757004]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d30_s1.json b/results/scale_v3_cifar10_dfa_w64_d30_s1.json new file mode 100644 index 0000000..626b568 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d30_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d30_s1", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.638411045074463, "cos_r_negg": [0.07048444449901581, -0.05516732856631279, 0.01678534597158432, -0.01635775715112686, 0.031921252608299255, -0.04854691028594971, -0.04836565628647804, 0.08578532934188843, 0.019566163420677185, 0.006045038346201181, 0.057466939091682434, -0.01568179577589035, -0.04699399322271347, -0.07758507132530212, 0.07345584034919739, 0.005482854321599007, 0.03354407846927643, -0.07221400737762451, 0.0355764701962471, 0.02614639699459076, -0.05729223042726517, -0.009635068476200104, -0.0016776907723397017, -0.02917255088686943, 0.006628268398344517, 0.07091523706912994, -0.03135240077972412, -0.018267279490828514, -0.0883563831448555, 0.03499498963356018], "cos_innovation_negg": [0.07048444449901581, -0.05516732856631279, 0.01678534597158432, -0.01635775715112686, 0.031921252608299255, -0.04854691028594971, -0.04836565628647804, 0.08578532934188843, 0.019566163420677185, 0.006045038346201181, 0.057466939091682434, -0.01568179577589035, -0.04699399322271347, -0.07758507132530212, 0.07345584034919739, 0.005482854321599007, 0.03354407846927643, -0.07221400737762451, 0.0355764701962471, 0.02614639699459076, -0.05729223042726517, -0.009635068476200104, -0.0016776907723397017, -0.02917255088686943, 0.006628268398344517, 0.07091523706912994, -0.03135240077972412, -0.018267279490828514, -0.0883563831448555, 0.03499498963356018], "cos_apical_negg": [0.07048444449901581, -0.05516732856631279, 0.01678534597158432, -0.01635775715112686, 0.031921252608299255, -0.04854691028594971, -0.04836565628647804, 0.08578532934188843, 0.019566163420677185, 0.006045038346201181, 0.057466939091682434, -0.01568179577589035, -0.04699399322271347, -0.07758507132530212, 0.07345584034919739, 0.005482854321599007, 0.03354407846927643, -0.07221400737762451, 0.0355764701962471, 0.02614639699459076, -0.05729223042726517, -0.009635068476200104, -0.0016776907723397017, -0.02917255088686943, 0.006628268398344517, 0.07091523706912994, -0.03135240077972412, -0.018267279490828514, -0.0883563831448555, 0.03499498963356018], "cos_Ac_negg": [0.07048444449901581, -0.05516732856631279, 0.01678534597158432, -0.01635775715112686, 0.031921252608299255, -0.04854691028594971, -0.04836565628647804, 0.08578532934188843, 0.019566163420677185, 0.006045038346201181, 0.057466939091682434, -0.01568179577589035, -0.04699399322271347, -0.07758507132530212, 0.07345584034919739, 0.005482854321599007, 0.03354407846927643, -0.07221400737762451, 0.0355764701962471, 0.02614639699459076, -0.05729223042726517, -0.009635068476200104, -0.0016776907723397017, -0.02917255088686943, 0.006628268398344517, 0.07091523706912994, -0.03135240077972412, -0.018267279490828514, -0.0883563831448555, 0.03499498963356018], "r_norm": [2.3714723587036133, 2.401730537414551, 2.4771108627319336, 2.4023666381835938, 2.628223180770874, 2.556312084197998, 2.636826992034912, 2.463465929031372, 2.448303699493408, 2.5774192810058594, 2.5303235054016113, 2.557281494140625, 2.402761459350586, 2.4862074851989746, 2.603732109069824, 2.399366855621338, 2.4296793937683105, 2.6283016204833984, 2.505570411682129, 2.532831907272339, 2.401393175125122, 2.5052378177642822, 2.6436123847961426, 2.547758102416992, 2.415070056915283, 2.391148567199707, 2.5781893730163574, 2.572446346282959, 2.4041597843170166, 2.4346494674682617]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3356, "test_loss": 1.88473427734375}, {"step": 500, "epoch": 1, "train_loss": 1.725431203842163, "cos_r_negg": [0.4966582953929901, 0.0016931581776589155, -0.052643924951553345, 0.06154335290193558, -0.07216247916221619, 0.012438920326530933, 0.0004024449735879898, 0.017102738842368126, 0.06742563843727112, -0.03785368800163269, 0.018652616068720818, 0.038536570966243744, 0.024521445855498314, 0.026736775413155556, 0.023032287135720253, -0.022096633911132812, -0.013665871694684029, 0.03545696288347244, 0.00793643482029438, 0.003359135240316391, 0.0340147390961647, 0.009730867110192776, 0.01809963770210743, -0.05517057329416275, 0.065555140376091, 0.06271353363990784, -0.04796028137207031, -0.026066165417432785, 0.03288397565484047, -0.022356927394866943], "cos_innovation_negg": [0.4966582953929901, 0.0016931581776589155, -0.052643924951553345, 0.06154335290193558, -0.07216247916221619, 0.012438920326530933, 0.0004024449735879898, 0.017102738842368126, 0.06742563843727112, -0.03785368800163269, 0.018652616068720818, 0.038536570966243744, 0.024521445855498314, 0.026736775413155556, 0.023032287135720253, -0.022096633911132812, -0.013665871694684029, 0.03545696288347244, 0.00793643482029438, 0.003359135240316391, 0.0340147390961647, 0.009730867110192776, 0.01809963770210743, -0.05517057329416275, 0.065555140376091, 0.06271353363990784, -0.04796028137207031, -0.026066165417432785, 0.03288397565484047, -0.022356927394866943], "cos_apical_negg": [0.4966582953929901, 0.0016931581776589155, -0.052643924951553345, 0.06154335290193558, -0.07216247916221619, 0.012438920326530933, 0.0004024449735879898, 0.017102738842368126, 0.06742563843727112, -0.03785368800163269, 0.018652616068720818, 0.038536570966243744, 0.024521445855498314, 0.026736775413155556, 0.023032287135720253, -0.022096633911132812, -0.013665871694684029, 0.03545696288347244, 0.00793643482029438, 0.003359135240316391, 0.0340147390961647, 0.009730867110192776, 0.01809963770210743, -0.05517057329416275, 0.065555140376091, 0.06271353363990784, -0.04796028137207031, -0.026066165417432785, 0.03288397565484047, -0.022356927394866943], "cos_Ac_negg": [0.4966582953929901, 0.0016931581776589155, -0.052643924951553345, 0.06154335290193558, -0.07216247916221619, 0.012438920326530933, 0.0004024449735879898, 0.017102738842368126, 0.06742563843727112, -0.03785368800163269, 0.018652616068720818, 0.038536570966243744, 0.024521445855498314, 0.026736775413155556, 0.023032287135720253, -0.022096633911132812, -0.013665871694684029, 0.03545696288347244, 0.00793643482029438, 0.003359135240316391, 0.0340147390961647, 0.009730867110192776, 0.01809963770210743, -0.05517057329416275, 0.065555140376091, 0.06271353363990784, -0.04796028137207031, -0.026066165417432785, 0.03288397565484047, -0.022356927394866943], "r_norm": [2.093005657196045, 2.097968578338623, 2.170957088470459, 2.1221799850463867, 2.2680039405822754, 2.266817331314087, 2.289808750152588, 2.155273914337158, 2.1568379402160645, 2.260167360305786, 2.22625994682312, 2.2358250617980957, 2.1122822761535645, 2.1859984397888184, 2.305304527282715, 2.114598274230957, 2.1352152824401855, 2.292222499847412, 2.2122364044189453, 2.1993141174316406, 2.1260199546813965, 2.207294225692749, 2.319164276123047, 2.250382900238037, 2.1269760131835938, 2.12129545211792, 2.2845711708068848, 2.2501564025878906, 2.1074790954589844, 2.156890392303467]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3607, "test_loss": 1.848004150390625}, {"step": 1000, "epoch": 2, "train_loss": 1.9055931568145752, "cos_r_negg": [0.5476503372192383, 0.012585821561515331, -0.036684636026620865, 0.06361765414476395, -0.0291905477643013, -0.04378458112478256, 0.04324546456336975, 0.016302574425935745, 0.05166435241699219, -0.06049654632806778, 0.04075143486261368, -0.04702010750770569, 0.02912922203540802, 0.026113446801900864, 0.10313498228788376, 0.014806590974330902, 0.0024662166833877563, 0.07009334862232208, 0.04857832193374634, -0.0145382359623909, 0.041769906878471375, 0.028640320524573326, 0.06851519644260406, 0.042375534772872925, 0.03191370517015457, 0.05285017937421799, -0.03270817548036575, 0.06886843591928482, 0.002595517784357071, 0.037176713347435], "cos_innovation_negg": [0.5476503372192383, 0.012585821561515331, -0.036684636026620865, 0.06361765414476395, -0.0291905477643013, -0.04378458112478256, 0.04324546456336975, 0.016302574425935745, 0.05166435241699219, -0.06049654632806778, 0.04075143486261368, -0.04702010750770569, 0.02912922203540802, 0.026113446801900864, 0.10313498228788376, 0.014806590974330902, 0.0024662166833877563, 0.07009334862232208, 0.04857832193374634, -0.0145382359623909, 0.041769906878471375, 0.028640320524573326, 0.06851519644260406, 0.042375534772872925, 0.03191370517015457, 0.05285017937421799, -0.03270817548036575, 0.06886843591928482, 0.002595517784357071, 0.037176713347435], "cos_apical_negg": [0.5476503372192383, 0.012585821561515331, -0.036684636026620865, 0.06361765414476395, -0.0291905477643013, -0.04378458112478256, 0.04324546456336975, 0.016302574425935745, 0.05166435241699219, -0.06049654632806778, 0.04075143486261368, -0.04702010750770569, 0.02912922203540802, 0.026113446801900864, 0.10313498228788376, 0.014806590974330902, 0.0024662166833877563, 0.07009334862232208, 0.04857832193374634, -0.0145382359623909, 0.041769906878471375, 0.028640320524573326, 0.06851519644260406, 0.042375534772872925, 0.03191370517015457, 0.05285017937421799, -0.03270817548036575, 0.06886843591928482, 0.002595517784357071, 0.037176713347435], "cos_Ac_negg": [0.5476503372192383, 0.012585821561515331, -0.036684636026620865, 0.06361765414476395, -0.0291905477643013, -0.04378458112478256, 0.04324546456336975, 0.016302574425935745, 0.05166435241699219, -0.06049654632806778, 0.04075143486261368, -0.04702010750770569, 0.02912922203540802, 0.026113446801900864, 0.10313498228788376, 0.014806590974330902, 0.0024662166833877563, 0.07009334862232208, 0.04857832193374634, -0.0145382359623909, 0.041769906878471375, 0.028640320524573326, 0.06851519644260406, 0.042375534772872925, 0.03191370517015457, 0.05285017937421799, -0.03270817548036575, 0.06886843591928482, 0.002595517784357071, 0.037176713347435], "r_norm": [2.0860865116119385, 2.0900561809539795, 2.1610000133514404, 2.123899459838867, 2.2521462440490723, 2.266068458557129, 2.282820463180542, 2.1465494632720947, 2.1478285789489746, 2.2444169521331787, 2.216182231903076, 2.2119462490081787, 2.10117506980896, 2.1821677684783936, 2.292414665222168, 2.1159615516662598, 2.130207061767578, 2.2827389240264893, 2.200930595397949, 2.19234299659729, 2.1202943325042725, 2.1965227127075195, 2.308171272277832, 2.2353475093841553, 2.1206231117248535, 2.1108322143554688, 2.2702126502990723, 2.2436766624450684, 2.0996196269989014, 2.1477394104003906]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3442, "test_loss": 1.8564998413085938}, {"step": 1500, "epoch": 3, "train_loss": 1.7237581014633179, "cos_r_negg": [0.5754849910736084, 0.04612713307142258, -0.0863955095410347, 0.026744958013296127, 0.01253366656601429, -0.014045648276805878, 0.10342328995466232, 0.032052990049123764, 0.08061209321022034, -0.02120249718427658, 0.09074169397354126, 0.06044008582830429, 0.0488494411110878, 0.054799824953079224, 0.11142852157354355, 0.02056385949254036, 0.0049476055428385735, 0.07189013063907623, 0.034604720771312714, -0.03651997819542885, 0.08967524766921997, 0.003980148583650589, 0.05659626051783562, -0.0006226068362593651, 0.05452972650527954, 0.04716978594660759, -0.04927561432123184, 0.013443139381706715, -0.017403962090611458, 0.015504611656069756], "cos_innovation_negg": [0.5754849910736084, 0.04612713307142258, -0.0863955095410347, 0.026744958013296127, 0.01253366656601429, -0.014045648276805878, 0.10342328995466232, 0.032052990049123764, 0.08061209321022034, -0.02120249718427658, 0.09074169397354126, 0.06044008582830429, 0.0488494411110878, 0.054799824953079224, 0.11142852157354355, 0.02056385949254036, 0.0049476055428385735, 0.07189013063907623, 0.034604720771312714, -0.03651997819542885, 0.08967524766921997, 0.003980148583650589, 0.05659626051783562, -0.0006226068362593651, 0.05452972650527954, 0.04716978594660759, -0.04927561432123184, 0.013443139381706715, -0.017403962090611458, 0.015504611656069756], "cos_apical_negg": [0.5754849910736084, 0.04612713307142258, -0.0863955095410347, 0.026744958013296127, 0.01253366656601429, -0.014045648276805878, 0.10342328995466232, 0.032052990049123764, 0.08061209321022034, -0.02120249718427658, 0.09074169397354126, 0.06044008582830429, 0.0488494411110878, 0.054799824953079224, 0.11142852157354355, 0.02056385949254036, 0.0049476055428385735, 0.07189013063907623, 0.034604720771312714, -0.03651997819542885, 0.08967524766921997, 0.003980148583650589, 0.05659626051783562, -0.0006226068362593651, 0.05452972650527954, 0.04716978594660759, -0.04927561432123184, 0.013443139381706715, -0.017403962090611458, 0.015504611656069756], "cos_Ac_negg": [0.5754849910736084, 0.04612713307142258, -0.0863955095410347, 0.026744958013296127, 0.01253366656601429, -0.014045648276805878, 0.10342328995466232, 0.032052990049123764, 0.08061209321022034, -0.02120249718427658, 0.09074169397354126, 0.06044008582830429, 0.0488494411110878, 0.054799824953079224, 0.11142852157354355, 0.02056385949254036, 0.0049476055428385735, 0.07189013063907623, 0.034604720771312714, -0.03651997819542885, 0.08967524766921997, 0.003980148583650589, 0.05659626051783562, -0.0006226068362593651, 0.05452972650527954, 0.04716978594660759, -0.04927561432123184, 0.013443139381706715, -0.017403962090611458, 0.015504611656069756], "r_norm": [2.0478272438049316, 2.0437588691711426, 2.1104812622070312, 2.0806050300598145, 2.201965808868408, 2.2167391777038574, 2.23866605758667, 2.1046364307403564, 2.101073741912842, 2.200453758239746, 2.172368049621582, 2.178766965866089, 2.06284499168396, 2.1403980255126953, 2.237375497817993, 2.065856695175171, 2.0855631828308105, 2.23989200592041, 2.153698444366455, 2.148697853088379, 2.077995777130127, 2.1472959518432617, 2.261932373046875, 2.1930675506591797, 2.07943058013916, 2.065424919128418, 2.2220497131347656, 2.1937012672424316, 2.060370445251465, 2.1045351028442383]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3646, "test_loss": 1.8111270874023437}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3607, "test_loss": 1.8225611450195311}], "final": {"test_acc": 0.3607, "test_loss": 1.8225611450195311, "wall_s": 23.19337224960327, "cos_r_negg": [0.5269464254379272, 0.047450169920921326, -0.0217910036444664, 0.06364326179027557, 0.009928952902555466, 0.016903642565011978, 0.11732660979032516, 0.06949736922979355, 0.12274371087551117, -0.042670838534832, 0.0518350750207901, 0.02584891952574253, 0.07354535907506943, 0.04735591262578964, 0.10072663426399231, 0.0209265798330307, 0.020462581887841225, 0.05780993402004242, 0.0317818745970726, 6.222259253263474e-05, 0.08149611949920654, 0.02124006487429142, 0.049032725393772125, 0.016597449779510498, 0.06690078228712082, 0.09298083931207657, -0.021555602550506592, 0.04722112417221069, 0.06447760760784149, 0.021421916782855988], "cos_innovation_negg": [0.5269464254379272, 0.047450169920921326, -0.0217910036444664, 0.06364326179027557, 0.009928952902555466, 0.016903642565011978, 0.11732660979032516, 0.06949736922979355, 0.12274371087551117, -0.042670838534832, 0.0518350750207901, 0.02584891952574253, 0.07354535907506943, 0.04735591262578964, 0.10072663426399231, 0.0209265798330307, 0.020462581887841225, 0.05780993402004242, 0.0317818745970726, 6.222259253263474e-05, 0.08149611949920654, 0.02124006487429142, 0.049032725393772125, 0.016597449779510498, 0.06690078228712082, 0.09298083931207657, -0.021555602550506592, 0.04722112417221069, 0.06447760760784149, 0.021421916782855988], "cos_apical_negg": [0.5269464254379272, 0.047450169920921326, -0.0217910036444664, 0.06364326179027557, 0.009928952902555466, 0.016903642565011978, 0.11732660979032516, 0.06949736922979355, 0.12274371087551117, -0.042670838534832, 0.0518350750207901, 0.02584891952574253, 0.07354535907506943, 0.04735591262578964, 0.10072663426399231, 0.0209265798330307, 0.020462581887841225, 0.05780993402004242, 0.0317818745970726, 6.222259253263474e-05, 0.08149611949920654, 0.02124006487429142, 0.049032725393772125, 0.016597449779510498, 0.06690078228712082, 0.09298083931207657, -0.021555602550506592, 0.04722112417221069, 0.06447760760784149, 0.021421916782855988], "cos_Ac_negg": [0.5269464254379272, 0.047450169920921326, -0.0217910036444664, 0.06364326179027557, 0.009928952902555466, 0.016903642565011978, 0.11732660979032516, 0.06949736922979355, 0.12274371087551117, -0.042670838534832, 0.0518350750207901, 0.02584891952574253, 0.07354535907506943, 0.04735591262578964, 0.10072663426399231, 0.0209265798330307, 0.020462581887841225, 0.05780993402004242, 0.0317818745970726, 6.222259253263474e-05, 0.08149611949920654, 0.02124006487429142, 0.049032725393772125, 0.016597449779510498, 0.06690078228712082, 0.09298083931207657, -0.021555602550506592, 0.04722112417221069, 0.06447760760784149, 0.021421916782855988]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d30_s2.json b/results/scale_v3_cifar10_dfa_w64_d30_s2.json new file mode 100644 index 0000000..0b13608 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d30_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d30_s2", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.7673251628875732, "cos_r_negg": [0.002249512355774641, 0.022959576919674873, -0.018467579036951065, 0.03803185746073723, -0.055734895169734955, 0.00045535527169704437, -0.07102607935667038, 0.010146564804017544, 0.02278125286102295, 0.0149698406457901, -0.005738881882280111, -0.06150812655687332, -0.037963781505823135, 0.0248748566955328, -0.04539640620350838, 0.027594605460762978, -0.02478851191699505, -0.08105120062828064, -0.09922529011964798, 0.0071421735920012, 0.0390196293592453, 0.08004271239042282, -0.05831489339470863, 0.060247354209423065, 0.018277838826179504, 0.03439311683177948, -0.015048985369503498, -0.024490593001246452, -0.015065079554915428, -0.009061385877430439], "cos_innovation_negg": [0.002249512355774641, 0.022959576919674873, -0.018467579036951065, 0.03803185746073723, -0.055734895169734955, 0.00045535527169704437, -0.07102607935667038, 0.010146564804017544, 0.02278125286102295, 0.0149698406457901, -0.005738881882280111, -0.06150812655687332, -0.037963781505823135, 0.0248748566955328, -0.04539640620350838, 0.027594605460762978, -0.02478851191699505, -0.08105120062828064, -0.09922529011964798, 0.0071421735920012, 0.0390196293592453, 0.08004271239042282, -0.05831489339470863, 0.060247354209423065, 0.018277838826179504, 0.03439311683177948, -0.015048985369503498, -0.024490593001246452, -0.015065079554915428, -0.009061385877430439], "cos_apical_negg": [0.002249512355774641, 0.022959576919674873, -0.018467579036951065, 0.03803185746073723, -0.055734895169734955, 0.00045535527169704437, -0.07102607935667038, 0.010146564804017544, 0.02278125286102295, 0.0149698406457901, -0.005738881882280111, -0.06150812655687332, -0.037963781505823135, 0.0248748566955328, -0.04539640620350838, 0.027594605460762978, -0.02478851191699505, -0.08105120062828064, -0.09922529011964798, 0.0071421735920012, 0.0390196293592453, 0.08004271239042282, -0.05831489339470863, 0.060247354209423065, 0.018277838826179504, 0.03439311683177948, -0.015048985369503498, -0.024490593001246452, -0.015065079554915428, -0.009061385877430439], "cos_Ac_negg": [0.002249512355774641, 0.022959576919674873, -0.018467579036951065, 0.03803185746073723, -0.055734895169734955, 0.00045535527169704437, -0.07102607935667038, 0.010146564804017544, 0.02278125286102295, 0.0149698406457901, -0.005738881882280111, -0.06150812655687332, -0.037963781505823135, 0.0248748566955328, -0.04539640620350838, 0.027594605460762978, -0.02478851191699505, -0.08105120062828064, -0.09922529011964798, 0.0071421735920012, 0.0390196293592453, 0.08004271239042282, -0.05831489339470863, 0.060247354209423065, 0.018277838826179504, 0.03439311683177948, -0.015048985369503498, -0.024490593001246452, -0.015065079554915428, -0.009061385877430439], "r_norm": [2.4331135749816895, 2.448417901992798, 2.3842456340789795, 2.4205610752105713, 2.3763790130615234, 2.485638380050659, 2.466935157775879, 2.5698516368865967, 2.603827476501465, 2.4623215198516846, 2.69050931930542, 2.4383158683776855, 2.399193286895752, 2.5776710510253906, 2.5124404430389404, 2.376943588256836, 2.3550431728363037, 2.4747495651245117, 2.427934169769287, 2.5004072189331055, 2.5258994102478027, 2.3911776542663574, 2.456050395965576, 2.4976251125335693, 2.4373555183410645, 2.593169689178467, 2.447528839111328, 2.4820315837860107, 2.4756150245666504, 2.4541056156158447]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3427, "test_loss": 1.8721033325195313}, {"step": 500, "epoch": 1, "train_loss": 1.798956036567688, "cos_r_negg": [0.4959384799003601, -0.005684107542037964, -0.03253506124019623, -0.017861198633909225, 0.0406477116048336, 0.06471378356218338, -0.025100424885749817, -0.007846258580684662, -0.04759366810321808, 0.0429055392742157, -0.07822134345769882, -0.08720242977142334, 0.08282889425754547, -0.007692225743085146, -0.032985106110572815, 0.011605325154960155, 0.06914891302585602, -0.10361462831497192, 0.030095210298895836, -0.04158181697130203, 0.022142386063933372, -0.05886797606945038, 0.058668993413448334, -0.005581153091043234, 0.01728830300271511, -0.025866732001304626, 0.06383432447910309, 0.039459601044654846, 0.006211932748556137, -0.011513623408973217], "cos_innovation_negg": [0.4959384799003601, -0.005684107542037964, -0.03253506124019623, -0.017861198633909225, 0.0406477116048336, 0.06471378356218338, -0.025100424885749817, -0.007846258580684662, -0.04759366810321808, 0.0429055392742157, -0.07822134345769882, -0.08720242977142334, 0.08282889425754547, -0.007692225743085146, -0.032985106110572815, 0.011605325154960155, 0.06914891302585602, -0.10361462831497192, 0.030095210298895836, -0.04158181697130203, 0.022142386063933372, -0.05886797606945038, 0.058668993413448334, -0.005581153091043234, 0.01728830300271511, -0.025866732001304626, 0.06383432447910309, 0.039459601044654846, 0.006211932748556137, -0.011513623408973217], "cos_apical_negg": [0.4959384799003601, -0.005684107542037964, -0.03253506124019623, -0.017861198633909225, 0.0406477116048336, 0.06471378356218338, -0.025100424885749817, -0.007846258580684662, -0.04759366810321808, 0.0429055392742157, -0.07822134345769882, -0.08720242977142334, 0.08282889425754547, -0.007692225743085146, -0.032985106110572815, 0.011605325154960155, 0.06914891302585602, -0.10361462831497192, 0.030095210298895836, -0.04158181697130203, 0.022142386063933372, -0.05886797606945038, 0.058668993413448334, -0.005581153091043234, 0.01728830300271511, -0.025866732001304626, 0.06383432447910309, 0.039459601044654846, 0.006211932748556137, -0.011513623408973217], "cos_Ac_negg": [0.4959384799003601, -0.005684107542037964, -0.03253506124019623, -0.017861198633909225, 0.0406477116048336, 0.06471378356218338, -0.025100424885749817, -0.007846258580684662, -0.04759366810321808, 0.0429055392742157, -0.07822134345769882, -0.08720242977142334, 0.08282889425754547, -0.007692225743085146, -0.032985106110572815, 0.011605325154960155, 0.06914891302585602, -0.10361462831497192, 0.030095210298895836, -0.04158181697130203, 0.022142386063933372, -0.05886797606945038, 0.058668993413448334, -0.005581153091043234, 0.01728830300271511, -0.025866732001304626, 0.06383432447910309, 0.039459601044654846, 0.006211932748556137, -0.011513623408973217], "r_norm": [2.137601375579834, 2.1617796421051025, 2.1047754287719727, 2.1368823051452637, 2.087362766265869, 2.1875364780426025, 2.178551435470581, 2.259371042251587, 2.274402618408203, 2.182623863220215, 2.3903069496154785, 2.1320977210998535, 2.120398998260498, 2.2514829635620117, 2.187145471572876, 2.1064372062683105, 2.0896291732788086, 2.1600897312164307, 2.1520042419433594, 2.2144150733947754, 2.2321019172668457, 2.128386974334717, 2.1819896697998047, 2.1784629821777344, 2.154487133026123, 2.3063125610351562, 2.1651957035064697, 2.188685894012451, 2.1694376468658447, 2.146679401397705]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3627, "test_loss": 1.8544914428710937}, {"step": 1000, "epoch": 2, "train_loss": 1.8124675750732422, "cos_r_negg": [0.5790170431137085, 0.0068262782879173756, 0.014878489077091217, -0.021209105849266052, -0.008565214462578297, 0.08287134021520615, -0.003738019149750471, 0.00792707409709692, -0.006075588054955006, 0.010389113798737526, 0.013715970329940319, -0.08543258160352707, 0.07382140308618546, -0.022730033844709396, -0.037367019802331924, 0.0459660068154335, 0.001288680825382471, -0.05667538568377495, 0.0035297502763569355, -0.011961514130234718, 0.003672307124361396, -0.09530255943536758, 0.044427625834941864, -0.018582481890916824, 0.05315515398979187, 0.014898143708705902, 0.07943236827850342, -0.04117360711097717, 0.04070630669593811, 0.02959546633064747], "cos_innovation_negg": [0.5790170431137085, 0.0068262782879173756, 0.014878489077091217, -0.021209105849266052, -0.008565214462578297, 0.08287134021520615, -0.003738019149750471, 0.00792707409709692, -0.006075588054955006, 0.010389113798737526, 0.013715970329940319, -0.08543258160352707, 0.07382140308618546, -0.022730033844709396, -0.037367019802331924, 0.0459660068154335, 0.001288680825382471, -0.05667538568377495, 0.0035297502763569355, -0.011961514130234718, 0.003672307124361396, -0.09530255943536758, 0.044427625834941864, -0.018582481890916824, 0.05315515398979187, 0.014898143708705902, 0.07943236827850342, -0.04117360711097717, 0.04070630669593811, 0.02959546633064747], "cos_apical_negg": [0.5790170431137085, 0.0068262782879173756, 0.014878489077091217, -0.021209105849266052, -0.008565214462578297, 0.08287134021520615, -0.003738019149750471, 0.00792707409709692, -0.006075588054955006, 0.010389113798737526, 0.013715970329940319, -0.08543258160352707, 0.07382140308618546, -0.022730033844709396, -0.037367019802331924, 0.0459660068154335, 0.001288680825382471, -0.05667538568377495, 0.0035297502763569355, -0.011961514130234718, 0.003672307124361396, -0.09530255943536758, 0.044427625834941864, -0.018582481890916824, 0.05315515398979187, 0.014898143708705902, 0.07943236827850342, -0.04117360711097717, 0.04070630669593811, 0.02959546633064747], "cos_Ac_negg": [0.5790170431137085, 0.0068262782879173756, 0.014878489077091217, -0.021209105849266052, -0.008565214462578297, 0.08287134021520615, -0.003738019149750471, 0.00792707409709692, -0.006075588054955006, 0.010389113798737526, 0.013715970329940319, -0.08543258160352707, 0.07382140308618546, -0.022730033844709396, -0.037367019802331924, 0.0459660068154335, 0.001288680825382471, -0.05667538568377495, 0.0035297502763569355, -0.011961514130234718, 0.003672307124361396, -0.09530255943536758, 0.044427625834941864, -0.018582481890916824, 0.05315515398979187, 0.014898143708705902, 0.07943236827850342, -0.04117360711097717, 0.04070630669593811, 0.02959546633064747], "r_norm": [2.1358258724212646, 2.153205394744873, 2.1028051376342773, 2.1363046169281006, 2.088613986968994, 2.19431209564209, 2.188533306121826, 2.267158269882202, 2.281900405883789, 2.187511920928955, 2.382333755493164, 2.130920886993408, 2.1255078315734863, 2.255626678466797, 2.2015626430511475, 2.0999929904937744, 2.087989091873169, 2.161282777786255, 2.1519885063171387, 2.209789752960205, 2.2398431301116943, 2.1251187324523926, 2.1790413856506348, 2.1789019107818604, 2.162919521331787, 2.3109803199768066, 2.1614503860473633, 2.188908100128174, 2.1768381595611572, 2.1339316368103027]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3539, "test_loss": 1.844827783203125}, {"step": 1500, "epoch": 3, "train_loss": 1.7862067222595215, "cos_r_negg": [0.5376938581466675, 0.017267026007175446, -0.006645498797297478, 0.00728624826297164, -0.016553815454244614, 0.10356226563453674, -0.028826354071497917, 0.053298044949769974, -0.02530461922287941, -0.00992338266223669, 0.01945805549621582, -0.05847847834229469, 0.0884273424744606, -0.02908310294151306, 0.01337914913892746, 0.07316087186336517, 0.019900145009160042, -0.021114975214004517, -0.022281864657998085, 0.028780994936823845, 0.05513715371489525, -0.09054272621870041, 0.10047529637813568, -0.010285192169249058, 0.0910220742225647, -0.018289636820554733, 0.06747397780418396, 0.01790446788072586, 0.07059430330991745, -0.03990055248141289], "cos_innovation_negg": [0.5376938581466675, 0.017267026007175446, -0.006645498797297478, 0.00728624826297164, -0.016553815454244614, 0.10356226563453674, -0.028826354071497917, 0.053298044949769974, -0.02530461922287941, -0.00992338266223669, 0.01945805549621582, -0.05847847834229469, 0.0884273424744606, -0.02908310294151306, 0.01337914913892746, 0.07316087186336517, 0.019900145009160042, -0.021114975214004517, -0.022281864657998085, 0.028780994936823845, 0.05513715371489525, -0.09054272621870041, 0.10047529637813568, -0.010285192169249058, 0.0910220742225647, -0.018289636820554733, 0.06747397780418396, 0.01790446788072586, 0.07059430330991745, -0.03990055248141289], "cos_apical_negg": [0.5376938581466675, 0.017267026007175446, -0.006645498797297478, 0.00728624826297164, -0.016553815454244614, 0.10356226563453674, -0.028826354071497917, 0.053298044949769974, -0.02530461922287941, -0.00992338266223669, 0.01945805549621582, -0.05847847834229469, 0.0884273424744606, -0.02908310294151306, 0.01337914913892746, 0.07316087186336517, 0.019900145009160042, -0.021114975214004517, -0.022281864657998085, 0.028780994936823845, 0.05513715371489525, -0.09054272621870041, 0.10047529637813568, -0.010285192169249058, 0.0910220742225647, -0.018289636820554733, 0.06747397780418396, 0.01790446788072586, 0.07059430330991745, -0.03990055248141289], "cos_Ac_negg": [0.5376938581466675, 0.017267026007175446, -0.006645498797297478, 0.00728624826297164, -0.016553815454244614, 0.10356226563453674, -0.028826354071497917, 0.053298044949769974, -0.02530461922287941, -0.00992338266223669, 0.01945805549621582, -0.05847847834229469, 0.0884273424744606, -0.02908310294151306, 0.01337914913892746, 0.07316087186336517, 0.019900145009160042, -0.021114975214004517, -0.022281864657998085, 0.028780994936823845, 0.05513715371489525, -0.09054272621870041, 0.10047529637813568, -0.010285192169249058, 0.0910220742225647, -0.018289636820554733, 0.06747397780418396, 0.01790446788072586, 0.07059430330991745, -0.03990055248141289], "r_norm": [2.1149516105651855, 2.1250386238098145, 2.0814123153686523, 2.1137514114379883, 2.0607430934906006, 2.168102741241455, 2.1653761863708496, 2.2294483184814453, 2.2551159858703613, 2.162332534790039, 2.3551862239837646, 2.1013975143432617, 2.095797300338745, 2.227856159210205, 2.1703944206237793, 2.0811104774475098, 2.0639243125915527, 2.1438937187194824, 2.130983352661133, 2.1887447834014893, 2.2018675804138184, 2.1081864833831787, 2.15849232673645, 2.1545753479003906, 2.130098819732666, 2.287865400314331, 2.1356403827667236, 2.164860725402832, 2.1414122581481934, 2.117335557937622]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3607, "test_loss": 1.8519111206054688}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3708, "test_loss": 1.8088109741210938}], "final": {"test_acc": 0.3708, "test_loss": 1.8088109741210938, "wall_s": 23.1907901763916, "cos_r_negg": [0.5909473896026611, 0.030654996633529663, -0.004572632722556591, 0.03871794044971466, 0.01197742111980915, 0.1383216381072998, 0.03996679186820984, 0.024300523102283478, -0.0037421202287077904, 0.06520158052444458, 0.0012239343486726284, -0.0599910207092762, 0.08771514147520065, 0.012411105446517467, 0.0019702515564858913, 0.014726400375366211, 0.0860552042722702, -0.04040795564651489, 0.039363324642181396, -0.009258367121219635, 0.07197955250740051, -0.07959756255149841, 0.05979757010936737, -0.02872471511363983, 0.12860217690467834, 0.045584384351968765, 0.08002200722694397, -0.03448985889554024, -0.015440849587321281, -0.0245670098811388], "cos_innovation_negg": [0.5909473896026611, 0.030654996633529663, -0.004572632722556591, 0.03871794044971466, 0.01197742111980915, 0.1383216381072998, 0.03996679186820984, 0.024300523102283478, -0.0037421202287077904, 0.06520158052444458, 0.0012239343486726284, -0.0599910207092762, 0.08771514147520065, 0.012411105446517467, 0.0019702515564858913, 0.014726400375366211, 0.0860552042722702, -0.04040795564651489, 0.039363324642181396, -0.009258367121219635, 0.07197955250740051, -0.07959756255149841, 0.05979757010936737, -0.02872471511363983, 0.12860217690467834, 0.045584384351968765, 0.08002200722694397, -0.03448985889554024, -0.015440849587321281, -0.0245670098811388], "cos_apical_negg": [0.5909473896026611, 0.030654996633529663, -0.004572632722556591, 0.03871794044971466, 0.01197742111980915, 0.1383216381072998, 0.03996679186820984, 0.024300523102283478, -0.0037421202287077904, 0.06520158052444458, 0.0012239343486726284, -0.0599910207092762, 0.08771514147520065, 0.012411105446517467, 0.0019702515564858913, 0.014726400375366211, 0.0860552042722702, -0.04040795564651489, 0.039363324642181396, -0.009258367121219635, 0.07197955250740051, -0.07959756255149841, 0.05979757010936737, -0.02872471511363983, 0.12860217690467834, 0.045584384351968765, 0.08002200722694397, -0.03448985889554024, -0.015440849587321281, -0.0245670098811388], "cos_Ac_negg": [0.5909473896026611, 0.030654996633529663, -0.004572632722556591, 0.03871794044971466, 0.01197742111980915, 0.1383216381072998, 0.03996679186820984, 0.024300523102283478, -0.0037421202287077904, 0.06520158052444458, 0.0012239343486726284, -0.0599910207092762, 0.08771514147520065, 0.012411105446517467, 0.0019702515564858913, 0.014726400375366211, 0.0860552042722702, -0.04040795564651489, 0.039363324642181396, -0.009258367121219635, 0.07197955250740051, -0.07959756255149841, 0.05979757010936737, -0.02872471511363983, 0.12860217690467834, 0.045584384351968765, 0.08002200722694397, -0.03448985889554024, -0.015440849587321281, -0.0245670098811388]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d30_s3.json b/results/scale_v3_cifar10_dfa_w64_d30_s3.json new file mode 100644 index 0000000..6c8cab4 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d30_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d30_s3", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.614835739135742, "cos_r_negg": [-0.0354691743850708, 0.011084317229688168, -0.03507746011018753, -0.04797753691673279, 0.02797923982143402, -0.03713028132915497, -0.007580794394016266, -0.031709469854831696, 0.03390783444046974, 0.06958069652318954, -0.004322929307818413, 0.0609910748898983, 0.0908372774720192, -0.02753886580467224, -0.05806727707386017, 0.1110665574669838, -0.02517387457191944, 0.06073632091283798, 0.002431281376630068, 0.004338130354881287, 0.07117453217506409, -0.03899887949228287, -0.048079777508974075, -0.017434734851121902, 0.04258080571889877, -0.04169657826423645, -0.003240067046135664, 0.030631093308329582, -0.007911516353487968, 0.0622338131070137], "cos_innovation_negg": [-0.0354691743850708, 0.011084317229688168, -0.03507746011018753, -0.04797753691673279, 0.02797923982143402, -0.03713028132915497, -0.007580794394016266, -0.031709469854831696, 0.03390783444046974, 0.06958069652318954, -0.004322929307818413, 0.0609910748898983, 0.0908372774720192, -0.02753886580467224, -0.05806727707386017, 0.1110665574669838, -0.02517387457191944, 0.06073632091283798, 0.002431281376630068, 0.004338130354881287, 0.07117453217506409, -0.03899887949228287, -0.048079777508974075, -0.017434734851121902, 0.04258080571889877, -0.04169657826423645, -0.003240067046135664, 0.030631093308329582, -0.007911516353487968, 0.0622338131070137], "cos_apical_negg": [-0.0354691743850708, 0.011084317229688168, -0.03507746011018753, -0.04797753691673279, 0.02797923982143402, -0.03713028132915497, -0.007580794394016266, -0.031709469854831696, 0.03390783444046974, 0.06958069652318954, -0.004322929307818413, 0.0609910748898983, 0.0908372774720192, -0.02753886580467224, -0.05806727707386017, 0.1110665574669838, -0.02517387457191944, 0.06073632091283798, 0.002431281376630068, 0.004338130354881287, 0.07117453217506409, -0.03899887949228287, -0.048079777508974075, -0.017434734851121902, 0.04258080571889877, -0.04169657826423645, -0.003240067046135664, 0.030631093308329582, -0.007911516353487968, 0.0622338131070137], "cos_Ac_negg": [-0.0354691743850708, 0.011084317229688168, -0.03507746011018753, -0.04797753691673279, 0.02797923982143402, -0.03713028132915497, -0.007580794394016266, -0.031709469854831696, 0.03390783444046974, 0.06958069652318954, -0.004322929307818413, 0.0609910748898983, 0.0908372774720192, -0.02753886580467224, -0.05806727707386017, 0.1110665574669838, -0.02517387457191944, 0.06073632091283798, 0.002431281376630068, 0.004338130354881287, 0.07117453217506409, -0.03899887949228287, -0.048079777508974075, -0.017434734851121902, 0.04258080571889877, -0.04169657826423645, -0.003240067046135664, 0.030631093308329582, -0.007911516353487968, 0.0622338131070137], "r_norm": [2.4395194053649902, 2.3524222373962402, 2.625286340713501, 2.4262709617614746, 2.3646240234375, 2.48673152923584, 2.5131940841674805, 2.494338274002075, 2.5015954971313477, 2.4734864234924316, 2.5628299713134766, 2.490208387374878, 2.3645966053009033, 2.444274425506592, 2.3756394386291504, 2.4287633895874023, 2.498383045196533, 2.440603017807007, 2.5153379440307617, 2.4611639976501465, 2.545836925506592, 2.52408504486084, 2.4028830528259277, 2.4840991497039795, 2.456599235534668, 2.429016590118408, 2.403512477874756, 2.5688743591308594, 2.510859966278076, 2.41801118850708]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3449, "test_loss": 1.8767222900390625}, {"step": 500, "epoch": 1, "train_loss": 1.8141272068023682, "cos_r_negg": [0.5834978818893433, 0.02172071300446987, -0.008782078512012959, -0.08165360242128372, 0.005386007949709892, 0.020413003861904144, 0.049750130623579025, -0.04050087928771973, 0.02649465575814247, 0.020961560308933258, 0.0035947985015809536, 0.048245564103126526, -0.10049033164978027, 0.049065954983234406, -0.000901503604836762, 0.04837929084897041, -0.02588057518005371, 0.01331876590847969, -0.03236357495188713, -0.030467335134744644, -0.0026370598934590816, 0.031721703708171844, 0.003576755989342928, -0.0738953948020935, 0.006696669384837151, -0.011140515096485615, 0.041674211621284485, 0.01569291017949581, -0.08862681686878204, -0.11423579603433609], "cos_innovation_negg": [0.5834978818893433, 0.02172071300446987, -0.008782078512012959, -0.08165360242128372, 0.005386007949709892, 0.020413003861904144, 0.049750130623579025, -0.04050087928771973, 0.02649465575814247, 0.020961560308933258, 0.0035947985015809536, 0.048245564103126526, -0.10049033164978027, 0.049065954983234406, -0.000901503604836762, 0.04837929084897041, -0.02588057518005371, 0.01331876590847969, -0.03236357495188713, -0.030467335134744644, -0.0026370598934590816, 0.031721703708171844, 0.003576755989342928, -0.0738953948020935, 0.006696669384837151, -0.011140515096485615, 0.041674211621284485, 0.01569291017949581, -0.08862681686878204, -0.11423579603433609], "cos_apical_negg": [0.5834978818893433, 0.02172071300446987, -0.008782078512012959, -0.08165360242128372, 0.005386007949709892, 0.020413003861904144, 0.049750130623579025, -0.04050087928771973, 0.02649465575814247, 0.020961560308933258, 0.0035947985015809536, 0.048245564103126526, -0.10049033164978027, 0.049065954983234406, -0.000901503604836762, 0.04837929084897041, -0.02588057518005371, 0.01331876590847969, -0.03236357495188713, -0.030467335134744644, -0.0026370598934590816, 0.031721703708171844, 0.003576755989342928, -0.0738953948020935, 0.006696669384837151, -0.011140515096485615, 0.041674211621284485, 0.01569291017949581, -0.08862681686878204, -0.11423579603433609], "cos_Ac_negg": [0.5834978818893433, 0.02172071300446987, -0.008782078512012959, -0.08165360242128372, 0.005386007949709892, 0.020413003861904144, 0.049750130623579025, -0.04050087928771973, 0.02649465575814247, 0.020961560308933258, 0.0035947985015809536, 0.048245564103126526, -0.10049033164978027, 0.049065954983234406, -0.000901503604836762, 0.04837929084897041, -0.02588057518005371, 0.01331876590847969, -0.03236357495188713, -0.030467335134744644, -0.0026370598934590816, 0.031721703708171844, 0.003576755989342928, -0.0738953948020935, 0.006696669384837151, -0.011140515096485615, 0.041674211621284485, 0.01569291017949581, -0.08862681686878204, -0.11423579603433609], "r_norm": [2.1641273498535156, 2.098332643508911, 2.3022866249084473, 2.142186164855957, 2.1220626831054688, 2.1967649459838867, 2.2501864433288574, 2.2486228942871094, 2.2339694499969482, 2.2253170013427734, 2.243060350418091, 2.22017502784729, 2.094665765762329, 2.2128593921661377, 2.1084399223327637, 2.1571812629699707, 2.2196788787841797, 2.1657493114471436, 2.2287704944610596, 2.2165684700012207, 2.25878643989563, 2.214918613433838, 2.115565776824951, 2.1991896629333496, 2.2043185234069824, 2.1633946895599365, 2.1089258193969727, 2.295809507369995, 2.225527286529541, 2.1454148292541504]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3592, "test_loss": 1.8489001098632813}, {"step": 1000, "epoch": 2, "train_loss": 1.8077439069747925, "cos_r_negg": [0.5426572561264038, -0.000471699982881546, -0.03370850905776024, -0.05805230885744095, 0.03651710972189903, 0.01930047571659088, -0.0027609760873019695, -0.057891279458999634, 0.060708366334438324, 0.010439660400152206, -0.0034087561070919037, 0.017884526401758194, -0.07258465886116028, 0.07702644914388657, 0.02569390833377838, 0.07109665870666504, 0.0006768857128918171, 0.03816009685397148, 0.01573631353676319, -0.009247805923223495, -0.0010661408305168152, 0.051315344870090485, 0.053149912506341934, -0.04660525545477867, -0.02670164406299591, -0.056551456451416016, 0.01708647795021534, 0.02116236463189125, -0.026996344327926636, -0.07165108621120453], "cos_innovation_negg": [0.5426572561264038, -0.000471699982881546, -0.03370850905776024, -0.05805230885744095, 0.03651710972189903, 0.01930047571659088, -0.0027609760873019695, -0.057891279458999634, 0.060708366334438324, 0.010439660400152206, -0.0034087561070919037, 0.017884526401758194, -0.07258465886116028, 0.07702644914388657, 0.02569390833377838, 0.07109665870666504, 0.0006768857128918171, 0.03816009685397148, 0.01573631353676319, -0.009247805923223495, -0.0010661408305168152, 0.051315344870090485, 0.053149912506341934, -0.04660525545477867, -0.02670164406299591, -0.056551456451416016, 0.01708647795021534, 0.02116236463189125, -0.026996344327926636, -0.07165108621120453], "cos_apical_negg": [0.5426572561264038, -0.000471699982881546, -0.03370850905776024, -0.05805230885744095, 0.03651710972189903, 0.01930047571659088, -0.0027609760873019695, -0.057891279458999634, 0.060708366334438324, 0.010439660400152206, -0.0034087561070919037, 0.017884526401758194, -0.07258465886116028, 0.07702644914388657, 0.02569390833377838, 0.07109665870666504, 0.0006768857128918171, 0.03816009685397148, 0.01573631353676319, -0.009247805923223495, -0.0010661408305168152, 0.051315344870090485, 0.053149912506341934, -0.04660525545477867, -0.02670164406299591, -0.056551456451416016, 0.01708647795021534, 0.02116236463189125, -0.026996344327926636, -0.07165108621120453], "cos_Ac_negg": [0.5426572561264038, -0.000471699982881546, -0.03370850905776024, -0.05805230885744095, 0.03651710972189903, 0.01930047571659088, -0.0027609760873019695, -0.057891279458999634, 0.060708366334438324, 0.010439660400152206, -0.0034087561070919037, 0.017884526401758194, -0.07258465886116028, 0.07702644914388657, 0.02569390833377838, 0.07109665870666504, 0.0006768857128918171, 0.03816009685397148, 0.01573631353676319, -0.009247805923223495, -0.0010661408305168152, 0.051315344870090485, 0.053149912506341934, -0.04660525545477867, -0.02670164406299591, -0.056551456451416016, 0.01708647795021534, 0.02116236463189125, -0.026996344327926636, -0.07165108621120453], "r_norm": [2.137204170227051, 2.069383144378662, 2.2760541439056396, 2.1043949127197266, 2.0947699546813965, 2.1611809730529785, 2.218050479888916, 2.218081474304199, 2.201650619506836, 2.1933698654174805, 2.197391986846924, 2.184309959411621, 2.070403575897217, 2.187321186065674, 2.073517322540283, 2.121030330657959, 2.1899051666259766, 2.1389708518981934, 2.1965551376342773, 2.180086135864258, 2.2175405025482178, 2.1903979778289795, 2.075549602508545, 2.173175811767578, 2.169332981109619, 2.1294052600860596, 2.0754308700561523, 2.265500068664551, 2.197822093963623, 2.107464075088501]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3481, "test_loss": 1.8497484985351562}, {"step": 1500, "epoch": 3, "train_loss": 1.724002480506897, "cos_r_negg": [0.5831311941146851, 0.00819560419768095, 0.014842926524579525, -0.04541250690817833, 0.020536597818136215, -0.006145058199763298, 0.015801813453435898, -0.011769660748541355, 0.09342478215694427, 0.050106413662433624, -0.06937626004219055, 0.029324382543563843, -0.017297446727752686, 0.07614302635192871, -0.040968939661979675, 0.08837269246578217, -0.013582034967839718, 0.05021585524082184, -0.020651238039135933, 0.04185253381729126, -0.03610069304704666, 0.033393409103155136, 0.08595992624759674, -0.08570028841495514, 0.0020112735219299793, -0.01912790909409523, 0.019765231758356094, 0.051920510828495026, -0.019497549161314964, -0.04156694561243057], "cos_innovation_negg": [0.5831311941146851, 0.00819560419768095, 0.014842926524579525, -0.04541250690817833, 0.020536597818136215, -0.006145058199763298, 0.015801813453435898, -0.011769660748541355, 0.09342478215694427, 0.050106413662433624, -0.06937626004219055, 0.029324382543563843, -0.017297446727752686, 0.07614302635192871, -0.040968939661979675, 0.08837269246578217, -0.013582034967839718, 0.05021585524082184, -0.020651238039135933, 0.04185253381729126, -0.03610069304704666, 0.033393409103155136, 0.08595992624759674, -0.08570028841495514, 0.0020112735219299793, -0.01912790909409523, 0.019765231758356094, 0.051920510828495026, -0.019497549161314964, -0.04156694561243057], "cos_apical_negg": [0.5831311941146851, 0.00819560419768095, 0.014842926524579525, -0.04541250690817833, 0.020536597818136215, -0.006145058199763298, 0.015801813453435898, -0.011769660748541355, 0.09342478215694427, 0.050106413662433624, -0.06937626004219055, 0.029324382543563843, -0.017297446727752686, 0.07614302635192871, -0.040968939661979675, 0.08837269246578217, -0.013582034967839718, 0.05021585524082184, -0.020651238039135933, 0.04185253381729126, -0.03610069304704666, 0.033393409103155136, 0.08595992624759674, -0.08570028841495514, 0.0020112735219299793, -0.01912790909409523, 0.019765231758356094, 0.051920510828495026, -0.019497549161314964, -0.04156694561243057], "cos_Ac_negg": [0.5831311941146851, 0.00819560419768095, 0.014842926524579525, -0.04541250690817833, 0.020536597818136215, -0.006145058199763298, 0.015801813453435898, -0.011769660748541355, 0.09342478215694427, 0.050106413662433624, -0.06937626004219055, 0.029324382543563843, -0.017297446727752686, 0.07614302635192871, -0.040968939661979675, 0.08837269246578217, -0.013582034967839718, 0.05021585524082184, -0.020651238039135933, 0.04185253381729126, -0.03610069304704666, 0.033393409103155136, 0.08595992624759674, -0.08570028841495514, 0.0020112735219299793, -0.01912790909409523, 0.019765231758356094, 0.051920510828495026, -0.019497549161314964, -0.04156694561243057], "r_norm": [2.148844003677368, 2.0736730098724365, 2.2784037590026855, 2.1243841648101807, 2.1072018146514893, 2.178985595703125, 2.2330057621002197, 2.232729434967041, 2.2185823917388916, 2.200716972351074, 2.2199649810791016, 2.2049193382263184, 2.0769972801208496, 2.1961281299591064, 2.084057569503784, 2.133882761001587, 2.204486608505249, 2.144850730895996, 2.216742515563965, 2.199141502380371, 2.2445669174194336, 2.2061338424682617, 2.096726417541504, 2.1802196502685547, 2.1895241737365723, 2.1474666595458984, 2.0943644046783447, 2.279433250427246, 2.2057342529296875, 2.125532627105713]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3656, "test_loss": 1.8275586059570312}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3733, "test_loss": 1.8026917602539063}], "final": {"test_acc": 0.3733, "test_loss": 1.8026917602539063, "wall_s": 22.949405431747437, "cos_r_negg": [0.602301836013794, 0.029450327157974243, 0.028264211490750313, -0.01377531886100769, -0.012301165610551834, 0.04648302122950554, 0.0061739846132695675, -0.04338932782411575, 0.08334119617938995, 0.04665948450565338, -0.05885539948940277, 0.07525753229856491, 0.006258122157305479, 0.053727999329566956, -0.030440522357821465, 0.11702220886945724, 0.013818957842886448, 0.06568506360054016, 0.02363518811762333, 0.044598422944545746, 0.0018955685663968325, 0.04365307092666626, 0.026811279356479645, -0.0560043603181839, 0.03428618982434273, -0.03289388120174408, 0.040004752576351166, 0.07738371193408966, -0.012301594950258732, -0.05684054270386696], "cos_innovation_negg": [0.602301836013794, 0.029450327157974243, 0.028264211490750313, -0.01377531886100769, -0.012301165610551834, 0.04648302122950554, 0.0061739846132695675, -0.04338932782411575, 0.08334119617938995, 0.04665948450565338, -0.05885539948940277, 0.07525753229856491, 0.006258122157305479, 0.053727999329566956, -0.030440522357821465, 0.11702220886945724, 0.013818957842886448, 0.06568506360054016, 0.02363518811762333, 0.044598422944545746, 0.0018955685663968325, 0.04365307092666626, 0.026811279356479645, -0.0560043603181839, 0.03428618982434273, -0.03289388120174408, 0.040004752576351166, 0.07738371193408966, -0.012301594950258732, -0.05684054270386696], "cos_apical_negg": [0.602301836013794, 0.029450327157974243, 0.028264211490750313, -0.01377531886100769, -0.012301165610551834, 0.04648302122950554, 0.0061739846132695675, -0.04338932782411575, 0.08334119617938995, 0.04665948450565338, -0.05885539948940277, 0.07525753229856491, 0.006258122157305479, 0.053727999329566956, -0.030440522357821465, 0.11702220886945724, 0.013818957842886448, 0.06568506360054016, 0.02363518811762333, 0.044598422944545746, 0.0018955685663968325, 0.04365307092666626, 0.026811279356479645, -0.0560043603181839, 0.03428618982434273, -0.03289388120174408, 0.040004752576351166, 0.07738371193408966, -0.012301594950258732, -0.05684054270386696], "cos_Ac_negg": [0.602301836013794, 0.029450327157974243, 0.028264211490750313, -0.01377531886100769, -0.012301165610551834, 0.04648302122950554, 0.0061739846132695675, -0.04338932782411575, 0.08334119617938995, 0.04665948450565338, -0.05885539948940277, 0.07525753229856491, 0.006258122157305479, 0.053727999329566956, -0.030440522357821465, 0.11702220886945724, 0.013818957842886448, 0.06568506360054016, 0.02363518811762333, 0.044598422944545746, 0.0018955685663968325, 0.04365307092666626, 0.026811279356479645, -0.0560043603181839, 0.03428618982434273, -0.03289388120174408, 0.040004752576351166, 0.07738371193408966, -0.012301594950258732, -0.05684054270386696]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d30_s4.json b/results/scale_v3_cifar10_dfa_w64_d30_s4.json new file mode 100644 index 0000000..9ac5b87 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d30_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d30_s4", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6514735221862793, "cos_r_negg": [-0.07412678748369217, -0.04394374042749405, 0.03722818195819855, -0.0717676505446434, -0.024675488471984863, -0.06367374956607819, 0.010915352031588554, -0.04810886085033417, 0.006848181132227182, 0.043644748628139496, -0.02917613834142685, 0.04982978105545044, 0.1312578171491623, 0.008211382664740086, -0.024526391178369522, -0.02007451467216015, -0.032842282205820084, 0.01009196974337101, 0.03763137012720108, -0.00727276923134923, -0.05193912610411644, 0.0058249859139323235, -0.031202469021081924, 0.04894702136516571, 0.03853687644004822, 0.05454086512327194, -0.019754357635974884, -0.0316607728600502, -0.0984528437256813, -0.011160830967128277], "cos_innovation_negg": [-0.07412678748369217, -0.04394374042749405, 0.03722818195819855, -0.0717676505446434, -0.024675488471984863, -0.06367374956607819, 0.010915352031588554, -0.04810886085033417, 0.006848181132227182, 0.043644748628139496, -0.02917613834142685, 0.04982978105545044, 0.1312578171491623, 0.008211382664740086, -0.024526391178369522, -0.02007451467216015, -0.032842282205820084, 0.01009196974337101, 0.03763137012720108, -0.00727276923134923, -0.05193912610411644, 0.0058249859139323235, -0.031202469021081924, 0.04894702136516571, 0.03853687644004822, 0.05454086512327194, -0.019754357635974884, -0.0316607728600502, -0.0984528437256813, -0.011160830967128277], "cos_apical_negg": [-0.07412678748369217, -0.04394374042749405, 0.03722818195819855, -0.0717676505446434, -0.024675488471984863, -0.06367374956607819, 0.010915352031588554, -0.04810886085033417, 0.006848181132227182, 0.043644748628139496, -0.02917613834142685, 0.04982978105545044, 0.1312578171491623, 0.008211382664740086, -0.024526391178369522, -0.02007451467216015, -0.032842282205820084, 0.01009196974337101, 0.03763137012720108, -0.00727276923134923, -0.05193912610411644, 0.0058249859139323235, -0.031202469021081924, 0.04894702136516571, 0.03853687644004822, 0.05454086512327194, -0.019754357635974884, -0.0316607728600502, -0.0984528437256813, -0.011160830967128277], "cos_Ac_negg": [-0.07412678748369217, -0.04394374042749405, 0.03722818195819855, -0.0717676505446434, -0.024675488471984863, -0.06367374956607819, 0.010915352031588554, -0.04810886085033417, 0.006848181132227182, 0.043644748628139496, -0.02917613834142685, 0.04982978105545044, 0.1312578171491623, 0.008211382664740086, -0.024526391178369522, -0.02007451467216015, -0.032842282205820084, 0.01009196974337101, 0.03763137012720108, -0.00727276923134923, -0.05193912610411644, 0.0058249859139323235, -0.031202469021081924, 0.04894702136516571, 0.03853687644004822, 0.05454086512327194, -0.019754357635974884, -0.0316607728600502, -0.0984528437256813, -0.011160830967128277], "r_norm": [2.5588512420654297, 2.570362091064453, 2.4625959396362305, 2.623915433883667, 2.481350898742676, 2.5674562454223633, 2.36057186126709, 2.4719841480255127, 2.523414373397827, 2.535059928894043, 2.419938564300537, 2.446058988571167, 2.2980198860168457, 2.439809799194336, 2.421278953552246, 2.5382442474365234, 2.49461030960083, 2.5223546028137207, 2.575709581375122, 2.4089722633361816, 2.4750900268554688, 2.604017972946167, 2.5957374572753906, 2.4383153915405273, 2.664546489715576, 2.6456892490386963, 2.4928691387176514, 2.4180941581726074, 2.6106534004211426, 2.520864486694336]}, {"epoch_end": 0, "step": 391, "test_acc": 0.348, "test_loss": 1.8706871337890625}, {"step": 500, "epoch": 1, "train_loss": 1.8439627885818481, "cos_r_negg": [0.4736138582229614, -0.0014424058608710766, 0.009048852138221264, 0.0004116976633667946, -0.037562981247901917, 0.012954644858837128, -0.047385863959789276, -0.010864355601370335, 0.033726539462804794, -0.0012843180447816849, -0.03284324333071709, 0.026365172117948532, -0.04277946054935455, 0.005492381285876036, 0.04753442108631134, -0.013697724789381027, -0.016441695392131805, -0.06219874322414398, -0.062301792204380035, -0.020501254126429558, 0.026943236589431763, -0.03881726413965225, -0.010959554463624954, 0.0033933939412236214, 0.007101545110344887, -0.033147092908620834, -0.0107832420617342, -0.08745378255844116, -0.06769201904535294, -0.004207614343613386], "cos_innovation_negg": [0.4736138582229614, -0.0014424058608710766, 0.009048852138221264, 0.0004116976633667946, -0.037562981247901917, 0.012954644858837128, -0.047385863959789276, -0.010864355601370335, 0.033726539462804794, -0.0012843180447816849, -0.03284324333071709, 0.026365172117948532, -0.04277946054935455, 0.005492381285876036, 0.04753442108631134, -0.013697724789381027, -0.016441695392131805, -0.06219874322414398, -0.062301792204380035, -0.020501254126429558, 0.026943236589431763, -0.03881726413965225, -0.010959554463624954, 0.0033933939412236214, 0.007101545110344887, -0.033147092908620834, -0.0107832420617342, -0.08745378255844116, -0.06769201904535294, -0.004207614343613386], "cos_apical_negg": [0.4736138582229614, -0.0014424058608710766, 0.009048852138221264, 0.0004116976633667946, -0.037562981247901917, 0.012954644858837128, -0.047385863959789276, -0.010864355601370335, 0.033726539462804794, -0.0012843180447816849, -0.03284324333071709, 0.026365172117948532, -0.04277946054935455, 0.005492381285876036, 0.04753442108631134, -0.013697724789381027, -0.016441695392131805, -0.06219874322414398, -0.062301792204380035, -0.020501254126429558, 0.026943236589431763, -0.03881726413965225, -0.010959554463624954, 0.0033933939412236214, 0.007101545110344887, -0.033147092908620834, -0.0107832420617342, -0.08745378255844116, -0.06769201904535294, -0.004207614343613386], "cos_Ac_negg": [0.4736138582229614, -0.0014424058608710766, 0.009048852138221264, 0.0004116976633667946, -0.037562981247901917, 0.012954644858837128, -0.047385863959789276, -0.010864355601370335, 0.033726539462804794, -0.0012843180447816849, -0.03284324333071709, 0.026365172117948532, -0.04277946054935455, 0.005492381285876036, 0.04753442108631134, -0.013697724789381027, -0.016441695392131805, -0.06219874322414398, -0.062301792204380035, -0.020501254126429558, 0.026943236589431763, -0.03881726413965225, -0.010959554463624954, 0.0033933939412236214, 0.007101545110344887, -0.033147092908620834, -0.0107832420617342, -0.08745378255844116, -0.06769201904535294, -0.004207614343613386], "r_norm": [2.209604263305664, 2.233380079269409, 2.1163787841796875, 2.280794143676758, 2.12677001953125, 2.21304988861084, 2.032297134399414, 2.155386209487915, 2.232576608657837, 2.1843552589416504, 2.0932188034057617, 2.1151649951934814, 2.017594337463379, 2.1009373664855957, 2.0948305130004883, 2.180410385131836, 2.1789028644561768, 2.178953170776367, 2.209444046020508, 2.1211507320404053, 2.150299072265625, 2.272489070892334, 2.2587947845458984, 2.1124706268310547, 2.297461748123169, 2.294257164001465, 2.1624627113342285, 2.058784008026123, 2.287343740463257, 2.1672091484069824]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3438, "test_loss": 1.86418115234375}, {"step": 1000, "epoch": 2, "train_loss": 1.8013699054718018, "cos_r_negg": [0.5449473857879639, 0.0741545706987381, -0.043738603591918945, 0.02429269254207611, -0.058084387332201004, 0.06871235370635986, 0.014279446564614773, -0.051156796514987946, 0.03927066549658775, 0.026014667004346848, -0.005130472593009472, 0.056468814611434937, -0.01695553958415985, 0.06928901374340057, 0.04100624844431877, -0.003490979317575693, 0.025237275287508965, -0.04323078691959381, 0.0415431410074234, -0.007112304214388132, -0.010452322661876678, -0.025781305506825447, 0.025241868570446968, 0.03980196639895439, 0.0609293058514595, -0.04992026090621948, 0.002512820065021515, -0.022370755672454834, -0.016954990103840828, 0.04724778234958649], "cos_innovation_negg": [0.5449473857879639, 0.0741545706987381, -0.043738603591918945, 0.02429269254207611, -0.058084387332201004, 0.06871235370635986, 0.014279446564614773, -0.051156796514987946, 0.03927066549658775, 0.026014667004346848, -0.005130472593009472, 0.056468814611434937, -0.01695553958415985, 0.06928901374340057, 0.04100624844431877, -0.003490979317575693, 0.025237275287508965, -0.04323078691959381, 0.0415431410074234, -0.007112304214388132, -0.010452322661876678, -0.025781305506825447, 0.025241868570446968, 0.03980196639895439, 0.0609293058514595, -0.04992026090621948, 0.002512820065021515, -0.022370755672454834, -0.016954990103840828, 0.04724778234958649], "cos_apical_negg": [0.5449473857879639, 0.0741545706987381, -0.043738603591918945, 0.02429269254207611, -0.058084387332201004, 0.06871235370635986, 0.014279446564614773, -0.051156796514987946, 0.03927066549658775, 0.026014667004346848, -0.005130472593009472, 0.056468814611434937, -0.01695553958415985, 0.06928901374340057, 0.04100624844431877, -0.003490979317575693, 0.025237275287508965, -0.04323078691959381, 0.0415431410074234, -0.007112304214388132, -0.010452322661876678, -0.025781305506825447, 0.025241868570446968, 0.03980196639895439, 0.0609293058514595, -0.04992026090621948, 0.002512820065021515, -0.022370755672454834, -0.016954990103840828, 0.04724778234958649], "cos_Ac_negg": [0.5449473857879639, 0.0741545706987381, -0.043738603591918945, 0.02429269254207611, -0.058084387332201004, 0.06871235370635986, 0.014279446564614773, -0.051156796514987946, 0.03927066549658775, 0.026014667004346848, -0.005130472593009472, 0.056468814611434937, -0.01695553958415985, 0.06928901374340057, 0.04100624844431877, -0.003490979317575693, 0.025237275287508965, -0.04323078691959381, 0.0415431410074234, -0.007112304214388132, -0.010452322661876678, -0.025781305506825447, 0.025241868570446968, 0.03980196639895439, 0.0609293058514595, -0.04992026090621948, 0.002512820065021515, -0.022370755672454834, -0.016954990103840828, 0.04724778234958649], "r_norm": [2.2193503379821777, 2.245670795440674, 2.1239542961120605, 2.2935075759887695, 2.1303861141204834, 2.2216737270355225, 2.038233757019043, 2.1628012657165527, 2.2394819259643555, 2.1977896690368652, 2.1020607948303223, 2.1312103271484375, 2.0248372554779053, 2.1112284660339355, 2.1055240631103516, 2.1817550659179688, 2.1848320960998535, 2.181640148162842, 2.2152810096740723, 2.134157657623291, 2.148752212524414, 2.2734148502349854, 2.263658046722412, 2.130530834197998, 2.3028860092163086, 2.3035175800323486, 2.1694130897521973, 2.0569803714752197, 2.2928056716918945, 2.1674301624298096]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3593, "test_loss": 1.82294287109375}, {"step": 1500, "epoch": 3, "train_loss": 1.7203726768493652, "cos_r_negg": [0.5224072337150574, 0.08232756704092026, 0.011276617646217346, 0.024618633091449738, -0.008408538065850735, 0.06281773746013641, 0.030908629298210144, 0.08687853068113327, 0.06202501803636551, 0.057719677686691284, 0.020303092896938324, 0.06768843531608582, -0.02782396413385868, 0.053834810853004456, 0.04473529011011124, 0.021032271906733513, 0.004771352745592594, -0.02674451470375061, -0.036392658948898315, -0.012178318575024605, 0.039869099855422974, 0.06599212437868118, 0.05632162094116211, 0.005896609276533127, 0.029705364257097244, -0.008051464334130287, 0.09016400575637817, -0.02407131716609001, -0.03864545002579689, -0.011006267741322517], "cos_innovation_negg": [0.5224072337150574, 0.08232756704092026, 0.011276617646217346, 0.024618633091449738, -0.008408538065850735, 0.06281773746013641, 0.030908629298210144, 0.08687853068113327, 0.06202501803636551, 0.057719677686691284, 0.020303092896938324, 0.06768843531608582, -0.02782396413385868, 0.053834810853004456, 0.04473529011011124, 0.021032271906733513, 0.004771352745592594, -0.02674451470375061, -0.036392658948898315, -0.012178318575024605, 0.039869099855422974, 0.06599212437868118, 0.05632162094116211, 0.005896609276533127, 0.029705364257097244, -0.008051464334130287, 0.09016400575637817, -0.02407131716609001, -0.03864545002579689, -0.011006267741322517], "cos_apical_negg": [0.5224072337150574, 0.08232756704092026, 0.011276617646217346, 0.024618633091449738, -0.008408538065850735, 0.06281773746013641, 0.030908629298210144, 0.08687853068113327, 0.06202501803636551, 0.057719677686691284, 0.020303092896938324, 0.06768843531608582, -0.02782396413385868, 0.053834810853004456, 0.04473529011011124, 0.021032271906733513, 0.004771352745592594, -0.02674451470375061, -0.036392658948898315, -0.012178318575024605, 0.039869099855422974, 0.06599212437868118, 0.05632162094116211, 0.005896609276533127, 0.029705364257097244, -0.008051464334130287, 0.09016400575637817, -0.02407131716609001, -0.03864545002579689, -0.011006267741322517], "cos_Ac_negg": [0.5224072337150574, 0.08232756704092026, 0.011276617646217346, 0.024618633091449738, -0.008408538065850735, 0.06281773746013641, 0.030908629298210144, 0.08687853068113327, 0.06202501803636551, 0.057719677686691284, 0.020303092896938324, 0.06768843531608582, -0.02782396413385868, 0.053834810853004456, 0.04473529011011124, 0.021032271906733513, 0.004771352745592594, -0.02674451470375061, -0.036392658948898315, -0.012178318575024605, 0.039869099855422974, 0.06599212437868118, 0.05632162094116211, 0.005896609276533127, 0.029705364257097244, -0.008051464334130287, 0.09016400575637817, -0.02407131716609001, -0.03864545002579689, -0.011006267741322517], "r_norm": [2.1704728603363037, 2.1999294757843018, 2.082566261291504, 2.2461535930633545, 2.0991005897521973, 2.175183057785034, 1.9969381093978882, 2.1179003715515137, 2.195991039276123, 2.1521432399749756, 2.0512542724609375, 2.0863161087036133, 1.9852945804595947, 2.0724334716796875, 2.0610620975494385, 2.142460823059082, 2.146230697631836, 2.1427111625671387, 2.1697444915771484, 2.0865073204040527, 2.109020948410034, 2.235302448272705, 2.231025457382202, 2.09555721282959, 2.260587692260742, 2.25747013092041, 2.130969285964966, 2.0235133171081543, 2.242058753967285, 2.118504524230957]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3594, "test_loss": 1.8305123168945312}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3715, "test_loss": 1.8011372680664062}], "final": {"test_acc": 0.3715, "test_loss": 1.8011372680664062, "wall_s": 24.686039447784424, "cos_r_negg": [0.539233386516571, 0.06221339479088783, 0.01457805186510086, 0.03833633288741112, 0.045453958213329315, 0.1271018534898758, -0.031142255291342735, 0.021093543618917465, 0.05619436502456665, 0.043654970824718475, -0.015611737966537476, 0.03791017830371857, -0.015462204813957214, 0.07092022895812988, 0.0653872862458229, 0.005964547395706177, -0.0049773999489843845, -0.028210220858454704, -0.03964021056890488, 0.028968922793865204, 0.05062485486268997, 0.0738900899887085, 0.06453075259923935, 0.03349278122186661, 0.04341408610343933, -0.027087397873401642, 0.049842193722724915, -0.018872590735554695, -0.013587675988674164, 0.026317965239286423], "cos_innovation_negg": [0.539233386516571, 0.06221339479088783, 0.01457805186510086, 0.03833633288741112, 0.045453958213329315, 0.1271018534898758, -0.031142255291342735, 0.021093543618917465, 0.05619436502456665, 0.043654970824718475, -0.015611737966537476, 0.03791017830371857, -0.015462204813957214, 0.07092022895812988, 0.0653872862458229, 0.005964547395706177, -0.0049773999489843845, -0.028210220858454704, -0.03964021056890488, 0.028968922793865204, 0.05062485486268997, 0.0738900899887085, 0.06453075259923935, 0.03349278122186661, 0.04341408610343933, -0.027087397873401642, 0.049842193722724915, -0.018872590735554695, -0.013587675988674164, 0.026317965239286423], "cos_apical_negg": [0.539233386516571, 0.06221339479088783, 0.01457805186510086, 0.03833633288741112, 0.045453958213329315, 0.1271018534898758, -0.031142255291342735, 0.021093543618917465, 0.05619436502456665, 0.043654970824718475, -0.015611737966537476, 0.03791017830371857, -0.015462204813957214, 0.07092022895812988, 0.0653872862458229, 0.005964547395706177, -0.0049773999489843845, -0.028210220858454704, -0.03964021056890488, 0.028968922793865204, 0.05062485486268997, 0.0738900899887085, 0.06453075259923935, 0.03349278122186661, 0.04341408610343933, -0.027087397873401642, 0.049842193722724915, -0.018872590735554695, -0.013587675988674164, 0.026317965239286423], "cos_Ac_negg": [0.539233386516571, 0.06221339479088783, 0.01457805186510086, 0.03833633288741112, 0.045453958213329315, 0.1271018534898758, -0.031142255291342735, 0.021093543618917465, 0.05619436502456665, 0.043654970824718475, -0.015611737966537476, 0.03791017830371857, -0.015462204813957214, 0.07092022895812988, 0.0653872862458229, 0.005964547395706177, -0.0049773999489843845, -0.028210220858454704, -0.03964021056890488, 0.028968922793865204, 0.05062485486268997, 0.0738900899887085, 0.06453075259923935, 0.03349278122186661, 0.04341408610343933, -0.027087397873401642, 0.049842193722724915, -0.018872590735554695, -0.013587675988674164, 0.026317965239286423]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d60_s0.json b/results/scale_v3_cifar10_dfa_w64_d60_s0.json new file mode 100644 index 0000000..a03d2b5 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d60_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d60_s0", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6154470443725586, "cos_r_negg": [-0.006716079078614712, 0.02438841015100479, -0.04525028541684151, 0.013871347531676292, -0.006345417816191912, -0.020512912422418594, -0.06818285584449768, -0.03158885985612869, -0.022064300253987312, -0.05176926404237747, 0.03123473934829235, -0.0071239229291677475, 0.0023502311669290066, -0.06361411511898041, -0.06265509873628616, -0.0626712515950203, 0.07430143654346466, 0.06538615375757217, 0.07140052318572998, -0.029584277421236038, 0.026476535946130753, -0.013373231515288353, 0.06404337286949158, 0.01492207683622837, -0.001631920924410224, -0.004349656403064728, -0.07081231474876404, -0.010206866078078747, -0.054892025887966156, -0.024155914783477783, 0.039311427623033524, -0.0015584426000714302, -0.028794849291443825, -0.028474921360611916, -0.008811524137854576, 0.002567646559327841, -0.002357651712372899, -0.013387875631451607, 0.034821949899196625, 0.011808709241449833, 0.10773257911205292, 0.027077380567789078, -0.02375342883169651, 0.04900947958230972, -0.033205289393663406, 0.003592877183109522, -0.045937854796648026, 0.0025787833146750927, -0.06160680949687958, 0.062074556946754456, -0.0015370987821370363, -0.09670576453208923, 0.05235959589481354, 0.026170790195465088, -0.02327905222773552, 0.05072954297065735, -0.0032390174455940723, -0.07727590948343277, -0.04464735835790634, -0.03091677464544773], "cos_innovation_negg": [-0.006716079078614712, 0.02438841015100479, -0.04525028541684151, 0.013871347531676292, -0.006345417816191912, -0.020512912422418594, -0.06818285584449768, -0.03158885985612869, -0.022064300253987312, -0.05176926404237747, 0.03123473934829235, -0.0071239229291677475, 0.0023502311669290066, -0.06361411511898041, -0.06265509873628616, -0.0626712515950203, 0.07430143654346466, 0.06538615375757217, 0.07140052318572998, -0.029584277421236038, 0.026476535946130753, -0.013373231515288353, 0.06404337286949158, 0.01492207683622837, -0.001631920924410224, -0.004349656403064728, -0.07081231474876404, -0.010206866078078747, -0.054892025887966156, -0.024155914783477783, 0.039311427623033524, -0.0015584426000714302, -0.028794849291443825, -0.028474921360611916, -0.008811524137854576, 0.002567646559327841, -0.002357651712372899, -0.013387875631451607, 0.034821949899196625, 0.011808709241449833, 0.10773257911205292, 0.027077380567789078, -0.02375342883169651, 0.04900947958230972, -0.033205289393663406, 0.003592877183109522, -0.045937854796648026, 0.0025787833146750927, -0.06160680949687958, 0.062074556946754456, -0.0015370987821370363, -0.09670576453208923, 0.05235959589481354, 0.026170790195465088, -0.02327905222773552, 0.05072954297065735, -0.0032390174455940723, -0.07727590948343277, -0.04464735835790634, -0.03091677464544773], "cos_apical_negg": [-0.006716079078614712, 0.02438841015100479, -0.04525028541684151, 0.013871347531676292, -0.006345417816191912, -0.020512912422418594, -0.06818285584449768, -0.03158885985612869, -0.022064300253987312, -0.05176926404237747, 0.03123473934829235, -0.0071239229291677475, 0.0023502311669290066, -0.06361411511898041, -0.06265509873628616, -0.0626712515950203, 0.07430143654346466, 0.06538615375757217, 0.07140052318572998, -0.029584277421236038, 0.026476535946130753, -0.013373231515288353, 0.06404337286949158, 0.01492207683622837, -0.001631920924410224, -0.004349656403064728, -0.07081231474876404, -0.010206866078078747, -0.054892025887966156, -0.024155914783477783, 0.039311427623033524, -0.0015584426000714302, -0.028794849291443825, -0.028474921360611916, -0.008811524137854576, 0.002567646559327841, -0.002357651712372899, -0.013387875631451607, 0.034821949899196625, 0.011808709241449833, 0.10773257911205292, 0.027077380567789078, -0.02375342883169651, 0.04900947958230972, -0.033205289393663406, 0.003592877183109522, -0.045937854796648026, 0.0025787833146750927, -0.06160680949687958, 0.062074556946754456, -0.0015370987821370363, -0.09670576453208923, 0.05235959589481354, 0.026170790195465088, -0.02327905222773552, 0.05072954297065735, -0.0032390174455940723, -0.07727590948343277, -0.04464735835790634, -0.03091677464544773], "cos_Ac_negg": [-0.006716079078614712, 0.02438841015100479, -0.04525028541684151, 0.013871347531676292, -0.006345417816191912, -0.020512912422418594, -0.06818285584449768, -0.03158885985612869, -0.022064300253987312, -0.05176926404237747, 0.03123473934829235, -0.0071239229291677475, 0.0023502311669290066, -0.06361411511898041, -0.06265509873628616, -0.0626712515950203, 0.07430143654346466, 0.06538615375757217, 0.07140052318572998, -0.029584277421236038, 0.026476535946130753, -0.013373231515288353, 0.06404337286949158, 0.01492207683622837, -0.001631920924410224, -0.004349656403064728, -0.07081231474876404, -0.010206866078078747, -0.054892025887966156, -0.024155914783477783, 0.039311427623033524, -0.0015584426000714302, -0.028794849291443825, -0.028474921360611916, -0.008811524137854576, 0.002567646559327841, -0.002357651712372899, -0.013387875631451607, 0.034821949899196625, 0.011808709241449833, 0.10773257911205292, 0.027077380567789078, -0.02375342883169651, 0.04900947958230972, -0.033205289393663406, 0.003592877183109522, -0.045937854796648026, 0.0025787833146750927, -0.06160680949687958, 0.062074556946754456, -0.0015370987821370363, -0.09670576453208923, 0.05235959589481354, 0.026170790195465088, -0.02327905222773552, 0.05072954297065735, -0.0032390174455940723, -0.07727590948343277, -0.04464735835790634, -0.03091677464544773], "r_norm": [2.476837158203125, 2.588827610015869, 2.507465362548828, 2.4364213943481445, 2.284212112426758, 2.463568687438965, 2.3808512687683105, 2.447174072265625, 2.5927276611328125, 2.5126051902770996, 2.4167494773864746, 2.584993839263916, 2.4760541915893555, 2.4647979736328125, 2.611776351928711, 2.5096240043640137, 2.580117702484131, 2.5247292518615723, 2.476726531982422, 2.5275793075561523, 2.617313861846924, 2.4330687522888184, 2.438448429107666, 2.5159451961517334, 2.457703113555908, 2.434385299682617, 2.6349055767059326, 2.4479267597198486, 2.3842978477478027, 2.6444690227508545, 2.3573412895202637, 2.4539451599121094, 2.4437050819396973, 2.4726710319519043, 2.5920939445495605, 2.5337939262390137, 2.3910603523254395, 2.3863282203674316, 2.5472004413604736, 2.4595766067504883, 2.525862455368042, 2.42592191696167, 2.493037223815918, 2.4348723888397217, 2.5188722610473633, 2.512610912322998, 2.5554099082946777, 2.4574851989746094, 2.4087142944335938, 2.429823875427246, 2.4081263542175293, 2.5474934577941895, 2.38215970993042, 2.451215982437134, 2.5639870166778564, 2.3927762508392334, 2.603609085083008, 2.484323024749756, 2.39697265625, 2.4256083965301514]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3403, "test_loss": 1.8708601318359375}, {"step": 500, "epoch": 1, "train_loss": 1.7038077116012573, "cos_r_negg": [0.5493764877319336, 0.01872267574071884, -0.05274730920791626, -0.0251530222594738, -0.02928505465388298, 0.021766304969787598, 0.010957082733511925, 0.02462802641093731, 0.007750675082206726, -0.03250172734260559, 0.013960735872387886, -0.03082090988755226, 0.021097920835018158, -0.027627158910036087, 0.027389727532863617, -0.005006357096135616, -0.04834308102726936, 0.01885084994137287, 0.019775716587901115, 0.0948190689086914, -0.061961255967617035, -0.017708592116832733, 0.01920778676867485, 0.05011627450585365, 0.12204429507255554, 0.04146486148238182, -0.030442122370004654, -0.0987292006611824, 0.040666453540325165, -0.04606577008962631, 0.041954271495342255, 0.07314084470272064, -0.0747339129447937, 0.029521167278289795, -0.04926997423171997, 0.02535322494804859, 0.025510795414447784, -0.0020328641403466463, -0.011173093691468239, -0.049767039716243744, -0.026263412088155746, 0.036890625953674316, -0.009711247868835926, -0.009342022240161896, 0.03803921863436699, 0.018719905987381935, 0.021647673100233078, 0.07000508904457092, -0.03713260591030121, -0.023778796195983887, -0.04196075350046158, -0.047701068222522736, -0.04460874944925308, 0.02074189856648445, 0.07091013342142105, -0.011496491730213165, 0.011327438987791538, 0.027212753891944885, -0.09790512174367905, -0.05965329706668854], "cos_innovation_negg": [0.5493764877319336, 0.01872267574071884, -0.05274730920791626, -0.0251530222594738, -0.02928505465388298, 0.021766304969787598, 0.010957082733511925, 0.02462802641093731, 0.007750675082206726, -0.03250172734260559, 0.013960735872387886, -0.03082090988755226, 0.021097920835018158, -0.027627158910036087, 0.027389727532863617, -0.005006357096135616, -0.04834308102726936, 0.01885084994137287, 0.019775716587901115, 0.0948190689086914, -0.061961255967617035, -0.017708592116832733, 0.01920778676867485, 0.05011627450585365, 0.12204429507255554, 0.04146486148238182, -0.030442122370004654, -0.0987292006611824, 0.040666453540325165, -0.04606577008962631, 0.041954271495342255, 0.07314084470272064, -0.0747339129447937, 0.029521167278289795, -0.04926997423171997, 0.02535322494804859, 0.025510795414447784, -0.0020328641403466463, -0.011173093691468239, -0.049767039716243744, -0.026263412088155746, 0.036890625953674316, -0.009711247868835926, -0.009342022240161896, 0.03803921863436699, 0.018719905987381935, 0.021647673100233078, 0.07000508904457092, -0.03713260591030121, -0.023778796195983887, -0.04196075350046158, -0.047701068222522736, -0.04460874944925308, 0.02074189856648445, 0.07091013342142105, -0.011496491730213165, 0.011327438987791538, 0.027212753891944885, -0.09790512174367905, -0.05965329706668854], "cos_apical_negg": [0.5493764877319336, 0.01872267574071884, -0.05274730920791626, -0.0251530222594738, -0.02928505465388298, 0.021766304969787598, 0.010957082733511925, 0.02462802641093731, 0.007750675082206726, -0.03250172734260559, 0.013960735872387886, -0.03082090988755226, 0.021097920835018158, -0.027627158910036087, 0.027389727532863617, -0.005006357096135616, -0.04834308102726936, 0.01885084994137287, 0.019775716587901115, 0.0948190689086914, -0.061961255967617035, -0.017708592116832733, 0.01920778676867485, 0.05011627450585365, 0.12204429507255554, 0.04146486148238182, -0.030442122370004654, -0.0987292006611824, 0.040666453540325165, -0.04606577008962631, 0.041954271495342255, 0.07314084470272064, -0.0747339129447937, 0.029521167278289795, -0.04926997423171997, 0.02535322494804859, 0.025510795414447784, -0.0020328641403466463, -0.011173093691468239, -0.049767039716243744, -0.026263412088155746, 0.036890625953674316, -0.009711247868835926, -0.009342022240161896, 0.03803921863436699, 0.018719905987381935, 0.021647673100233078, 0.07000508904457092, -0.03713260591030121, -0.023778796195983887, -0.04196075350046158, -0.047701068222522736, -0.04460874944925308, 0.02074189856648445, 0.07091013342142105, -0.011496491730213165, 0.011327438987791538, 0.027212753891944885, -0.09790512174367905, -0.05965329706668854], "cos_Ac_negg": [0.5493764877319336, 0.01872267574071884, -0.05274730920791626, -0.0251530222594738, -0.02928505465388298, 0.021766304969787598, 0.010957082733511925, 0.02462802641093731, 0.007750675082206726, -0.03250172734260559, 0.013960735872387886, -0.03082090988755226, 0.021097920835018158, -0.027627158910036087, 0.027389727532863617, -0.005006357096135616, -0.04834308102726936, 0.01885084994137287, 0.019775716587901115, 0.0948190689086914, -0.061961255967617035, -0.017708592116832733, 0.01920778676867485, 0.05011627450585365, 0.12204429507255554, 0.04146486148238182, -0.030442122370004654, -0.0987292006611824, 0.040666453540325165, -0.04606577008962631, 0.041954271495342255, 0.07314084470272064, -0.0747339129447937, 0.029521167278289795, -0.04926997423171997, 0.02535322494804859, 0.025510795414447784, -0.0020328641403466463, -0.011173093691468239, -0.049767039716243744, -0.026263412088155746, 0.036890625953674316, -0.009711247868835926, -0.009342022240161896, 0.03803921863436699, 0.018719905987381935, 0.021647673100233078, 0.07000508904457092, -0.03713260591030121, -0.023778796195983887, -0.04196075350046158, -0.047701068222522736, -0.04460874944925308, 0.02074189856648445, 0.07091013342142105, -0.011496491730213165, 0.011327438987791538, 0.027212753891944885, -0.09790512174367905, -0.05965329706668854], "r_norm": [2.1301116943359375, 2.232585906982422, 2.188831090927124, 2.1164536476135254, 1.99443781375885, 2.120631694793701, 2.0384206771850586, 2.1060214042663574, 2.2730677127838135, 2.1715474128723145, 2.1050007343292236, 2.220019578933716, 2.1526613235473633, 2.125145673751831, 2.2568631172180176, 2.1785054206848145, 2.227285385131836, 2.1672420501708984, 2.153294563293457, 2.2008848190307617, 2.291428327560425, 2.1129469871520996, 2.1159205436706543, 2.175632953643799, 2.148427963256836, 2.105003833770752, 2.2936763763427734, 2.1252517700195312, 2.0705881118774414, 2.286167621612549, 2.0095531940460205, 2.133378028869629, 2.109151601791382, 2.1417274475097656, 2.236701011657715, 2.1987481117248535, 2.0703554153442383, 2.081859588623047, 2.213632583618164, 2.1323516368865967, 2.211033344268799, 2.11452317237854, 2.1567158699035645, 2.117455005645752, 2.162360668182373, 2.1912946701049805, 2.211956024169922, 2.150634288787842, 2.105151891708374, 2.104519844055176, 2.1026124954223633, 2.2414767742156982, 2.0929369926452637, 2.1091196537017822, 2.2163383960723877, 2.0973386764526367, 2.248466730117798, 2.1563684940338135, 2.1060895919799805, 2.109330177307129]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3585, "test_loss": 1.8441361083984376}, {"step": 1000, "epoch": 2, "train_loss": 1.857997179031372, "cos_r_negg": [0.5343904495239258, 0.023018933832645416, -0.08571218699216843, -0.017818020656704903, -0.013029110617935658, 0.0748366266489029, 0.009163716807961464, 0.018014952540397644, 0.03471120819449425, 0.062045302242040634, 0.024813279509544373, -0.018575729802250862, 0.04900572821497917, -0.09401875734329224, 0.037270475178956985, -0.04430871456861496, -0.10339285433292389, 0.007345324382185936, -0.04356004297733307, 0.06882297992706299, -0.05566791445016861, 0.03943360969424248, 0.01875373348593712, 0.08718167245388031, 0.04017236828804016, -0.008893324993550777, 0.0021896613761782646, -0.005950205959379673, -0.009089072234928608, -0.007385949604213238, 0.08017701655626297, 0.027578268200159073, -0.06880444288253784, -0.007292254827916622, -0.015137006528675556, 0.049430668354034424, 0.059146419167518616, 0.018993591889739037, -0.0705374926328659, -0.02901827171444893, 0.0656881332397461, 0.014640504494309425, -0.024641098454594612, 0.03142322599887848, 0.02560984343290329, 0.018047764897346497, 0.022018013522028923, 0.019986581057310104, -0.0501948744058609, -0.03323870897293091, -0.06274554878473282, -0.041777465492486954, -0.04333692044019699, 0.02924686297774315, 0.04845748096704483, 0.02712920866906643, -0.02585148625075817, 0.03291165083646774, -0.044675443321466446, -0.06536038219928741], "cos_innovation_negg": [0.5343904495239258, 0.023018933832645416, -0.08571218699216843, -0.017818020656704903, -0.013029110617935658, 0.0748366266489029, 0.009163716807961464, 0.018014952540397644, 0.03471120819449425, 0.062045302242040634, 0.024813279509544373, -0.018575729802250862, 0.04900572821497917, -0.09401875734329224, 0.037270475178956985, -0.04430871456861496, -0.10339285433292389, 0.007345324382185936, -0.04356004297733307, 0.06882297992706299, -0.05566791445016861, 0.03943360969424248, 0.01875373348593712, 0.08718167245388031, 0.04017236828804016, -0.008893324993550777, 0.0021896613761782646, -0.005950205959379673, -0.009089072234928608, -0.007385949604213238, 0.08017701655626297, 0.027578268200159073, -0.06880444288253784, -0.007292254827916622, -0.015137006528675556, 0.049430668354034424, 0.059146419167518616, 0.018993591889739037, -0.0705374926328659, -0.02901827171444893, 0.0656881332397461, 0.014640504494309425, -0.024641098454594612, 0.03142322599887848, 0.02560984343290329, 0.018047764897346497, 0.022018013522028923, 0.019986581057310104, -0.0501948744058609, -0.03323870897293091, -0.06274554878473282, -0.041777465492486954, -0.04333692044019699, 0.02924686297774315, 0.04845748096704483, 0.02712920866906643, -0.02585148625075817, 0.03291165083646774, -0.044675443321466446, -0.06536038219928741], "cos_apical_negg": [0.5343904495239258, 0.023018933832645416, -0.08571218699216843, -0.017818020656704903, -0.013029110617935658, 0.0748366266489029, 0.009163716807961464, 0.018014952540397644, 0.03471120819449425, 0.062045302242040634, 0.024813279509544373, -0.018575729802250862, 0.04900572821497917, -0.09401875734329224, 0.037270475178956985, -0.04430871456861496, -0.10339285433292389, 0.007345324382185936, -0.04356004297733307, 0.06882297992706299, -0.05566791445016861, 0.03943360969424248, 0.01875373348593712, 0.08718167245388031, 0.04017236828804016, -0.008893324993550777, 0.0021896613761782646, -0.005950205959379673, -0.009089072234928608, -0.007385949604213238, 0.08017701655626297, 0.027578268200159073, -0.06880444288253784, -0.007292254827916622, -0.015137006528675556, 0.049430668354034424, 0.059146419167518616, 0.018993591889739037, -0.0705374926328659, -0.02901827171444893, 0.0656881332397461, 0.014640504494309425, -0.024641098454594612, 0.03142322599887848, 0.02560984343290329, 0.018047764897346497, 0.022018013522028923, 0.019986581057310104, -0.0501948744058609, -0.03323870897293091, -0.06274554878473282, -0.041777465492486954, -0.04333692044019699, 0.02924686297774315, 0.04845748096704483, 0.02712920866906643, -0.02585148625075817, 0.03291165083646774, -0.044675443321466446, -0.06536038219928741], "cos_Ac_negg": [0.5343904495239258, 0.023018933832645416, -0.08571218699216843, -0.017818020656704903, -0.013029110617935658, 0.0748366266489029, 0.009163716807961464, 0.018014952540397644, 0.03471120819449425, 0.062045302242040634, 0.024813279509544373, -0.018575729802250862, 0.04900572821497917, -0.09401875734329224, 0.037270475178956985, -0.04430871456861496, -0.10339285433292389, 0.007345324382185936, -0.04356004297733307, 0.06882297992706299, -0.05566791445016861, 0.03943360969424248, 0.01875373348593712, 0.08718167245388031, 0.04017236828804016, -0.008893324993550777, 0.0021896613761782646, -0.005950205959379673, -0.009089072234928608, -0.007385949604213238, 0.08017701655626297, 0.027578268200159073, -0.06880444288253784, -0.007292254827916622, -0.015137006528675556, 0.049430668354034424, 0.059146419167518616, 0.018993591889739037, -0.0705374926328659, -0.02901827171444893, 0.0656881332397461, 0.014640504494309425, -0.024641098454594612, 0.03142322599887848, 0.02560984343290329, 0.018047764897346497, 0.022018013522028923, 0.019986581057310104, -0.0501948744058609, -0.03323870897293091, -0.06274554878473282, -0.041777465492486954, -0.04333692044019699, 0.02924686297774315, 0.04845748096704483, 0.02712920866906643, -0.02585148625075817, 0.03291165083646774, -0.044675443321466446, -0.06536038219928741], "r_norm": [2.1409201622009277, 2.250765800476074, 2.2026870250701904, 2.1370391845703125, 2.0008046627044678, 2.1356072425842285, 2.059896945953369, 2.1249306201934814, 2.3059473037719727, 2.190365791320801, 2.12729811668396, 2.236894130706787, 2.168992042541504, 2.1360654830932617, 2.273104667663574, 2.197073459625244, 2.2362804412841797, 2.18412184715271, 2.1714978218078613, 2.222852945327759, 2.306452512741089, 2.132624387741089, 2.1374709606170654, 2.1788957118988037, 2.1593823432922363, 2.1206912994384766, 2.309680700302124, 2.147827625274658, 2.0874738693237305, 2.3094091415405273, 2.0352487564086914, 2.154064178466797, 2.1215386390686035, 2.1628928184509277, 2.253443956375122, 2.2141261100769043, 2.0794405937194824, 2.100123405456543, 2.235599994659424, 2.1525380611419678, 2.226761817932129, 2.1172890663146973, 2.1764984130859375, 2.138383626937866, 2.1824545860290527, 2.203514575958252, 2.224557399749756, 2.1673953533172607, 2.114952564239502, 2.128477096557617, 2.1164591312408447, 2.2617340087890625, 2.1204280853271484, 2.123225212097168, 2.2259621620178223, 2.117644786834717, 2.257298231124878, 2.171720504760742, 2.1138129234313965, 2.1265673637390137]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.346, "test_loss": 1.8643386474609376}, {"step": 1500, "epoch": 3, "train_loss": 1.8078371286392212, "cos_r_negg": [0.5321294665336609, 0.05931701511144638, -0.004795122426003218, -0.04091104492545128, -0.004944495391100645, 0.0733313038945198, 0.028732454404234886, -0.028997357934713364, 0.019572827965021133, -0.006624255329370499, 0.04258586838841438, -0.03980748727917671, 0.02592821791768074, -0.06463976949453354, 0.01925666071474552, -0.034768976271152496, -0.04177805036306381, -0.02030637487769127, -0.005816735327243805, 0.1192202940583229, -0.006012198515236378, 0.04902844876050949, 0.047489654272794724, 0.10890227556228638, 0.1260370910167694, 0.032090477645397186, 0.02169676497578621, -0.04004015773534775, 0.0039202249608933926, -0.024180863052606583, 0.027383621782064438, 0.061523422598838806, -0.08911024034023285, -0.03881463408470154, -0.004562738351523876, 0.06938697397708893, 0.045063890516757965, 0.024196378886699677, -0.024422064423561096, -0.01611250638961792, 0.07983332872390747, 0.03923346474766731, -0.016006378456950188, 0.01824045181274414, 0.0027705919928848743, -0.0370522066950798, 0.021477311849594116, 0.04933709278702736, -0.023096324875950813, -0.08095914870500565, -0.026619909331202507, -0.037193916738033295, -0.04080085828900337, 0.05228857323527336, 0.07540442049503326, 0.046373844146728516, 0.02178887277841568, 0.02086247317492962, -0.001101955771446228, -0.020234135910868645], "cos_innovation_negg": [0.5321294665336609, 0.05931701511144638, -0.004795122426003218, -0.04091104492545128, -0.004944495391100645, 0.0733313038945198, 0.028732454404234886, -0.028997357934713364, 0.019572827965021133, -0.006624255329370499, 0.04258586838841438, -0.03980748727917671, 0.02592821791768074, -0.06463976949453354, 0.01925666071474552, -0.034768976271152496, -0.04177805036306381, -0.02030637487769127, -0.005816735327243805, 0.1192202940583229, -0.006012198515236378, 0.04902844876050949, 0.047489654272794724, 0.10890227556228638, 0.1260370910167694, 0.032090477645397186, 0.02169676497578621, -0.04004015773534775, 0.0039202249608933926, -0.024180863052606583, 0.027383621782064438, 0.061523422598838806, -0.08911024034023285, -0.03881463408470154, -0.004562738351523876, 0.06938697397708893, 0.045063890516757965, 0.024196378886699677, -0.024422064423561096, -0.01611250638961792, 0.07983332872390747, 0.03923346474766731, -0.016006378456950188, 0.01824045181274414, 0.0027705919928848743, -0.0370522066950798, 0.021477311849594116, 0.04933709278702736, -0.023096324875950813, -0.08095914870500565, -0.026619909331202507, -0.037193916738033295, -0.04080085828900337, 0.05228857323527336, 0.07540442049503326, 0.046373844146728516, 0.02178887277841568, 0.02086247317492962, -0.001101955771446228, -0.020234135910868645], "cos_apical_negg": [0.5321294665336609, 0.05931701511144638, -0.004795122426003218, -0.04091104492545128, -0.004944495391100645, 0.0733313038945198, 0.028732454404234886, -0.028997357934713364, 0.019572827965021133, -0.006624255329370499, 0.04258586838841438, -0.03980748727917671, 0.02592821791768074, -0.06463976949453354, 0.01925666071474552, -0.034768976271152496, -0.04177805036306381, -0.02030637487769127, -0.005816735327243805, 0.1192202940583229, -0.006012198515236378, 0.04902844876050949, 0.047489654272794724, 0.10890227556228638, 0.1260370910167694, 0.032090477645397186, 0.02169676497578621, -0.04004015773534775, 0.0039202249608933926, -0.024180863052606583, 0.027383621782064438, 0.061523422598838806, -0.08911024034023285, -0.03881463408470154, -0.004562738351523876, 0.06938697397708893, 0.045063890516757965, 0.024196378886699677, -0.024422064423561096, -0.01611250638961792, 0.07983332872390747, 0.03923346474766731, -0.016006378456950188, 0.01824045181274414, 0.0027705919928848743, -0.0370522066950798, 0.021477311849594116, 0.04933709278702736, -0.023096324875950813, -0.08095914870500565, -0.026619909331202507, -0.037193916738033295, -0.04080085828900337, 0.05228857323527336, 0.07540442049503326, 0.046373844146728516, 0.02178887277841568, 0.02086247317492962, -0.001101955771446228, -0.020234135910868645], "cos_Ac_negg": [0.5321294665336609, 0.05931701511144638, -0.004795122426003218, -0.04091104492545128, -0.004944495391100645, 0.0733313038945198, 0.028732454404234886, -0.028997357934713364, 0.019572827965021133, -0.006624255329370499, 0.04258586838841438, -0.03980748727917671, 0.02592821791768074, -0.06463976949453354, 0.01925666071474552, -0.034768976271152496, -0.04177805036306381, -0.02030637487769127, -0.005816735327243805, 0.1192202940583229, -0.006012198515236378, 0.04902844876050949, 0.047489654272794724, 0.10890227556228638, 0.1260370910167694, 0.032090477645397186, 0.02169676497578621, -0.04004015773534775, 0.0039202249608933926, -0.024180863052606583, 0.027383621782064438, 0.061523422598838806, -0.08911024034023285, -0.03881463408470154, -0.004562738351523876, 0.06938697397708893, 0.045063890516757965, 0.024196378886699677, -0.024422064423561096, -0.01611250638961792, 0.07983332872390747, 0.03923346474766731, -0.016006378456950188, 0.01824045181274414, 0.0027705919928848743, -0.0370522066950798, 0.021477311849594116, 0.04933709278702736, -0.023096324875950813, -0.08095914870500565, -0.026619909331202507, -0.037193916738033295, -0.04080085828900337, 0.05228857323527336, 0.07540442049503326, 0.046373844146728516, 0.02178887277841568, 0.02086247317492962, -0.001101955771446228, -0.020234135910868645], "r_norm": [2.160775661468506, 2.2687463760375977, 2.221665382385254, 2.153689384460449, 2.0242919921875, 2.1568546295166016, 2.0728518962860107, 2.1425580978393555, 2.307276964187622, 2.2065024375915527, 2.14194393157959, 2.2507224082946777, 2.176478385925293, 2.160116195678711, 2.2926833629608154, 2.2115936279296875, 2.2673709392547607, 2.200784206390381, 2.185403823852539, 2.24324893951416, 2.3234622478485107, 2.1513333320617676, 2.1534721851348877, 2.2044684886932373, 2.1821813583374023, 2.134456157684326, 2.3281219005584717, 2.1578588485717773, 2.1006298065185547, 2.3158438205718994, 2.0407919883728027, 2.1536338329315186, 2.1423592567443848, 2.179675817489624, 2.270850658416748, 2.22934627532959, 2.09849214553833, 2.1187362670898438, 2.2497034072875977, 2.1682066917419434, 2.2457528114318848, 2.1450448036193848, 2.1872382164001465, 2.1476597785949707, 2.1952481269836426, 2.226612091064453, 2.24898624420166, 2.1792969703674316, 2.1390697956085205, 2.144273281097412, 2.1311168670654297, 2.27933931350708, 2.126697063446045, 2.1475963592529297, 2.245152711868286, 2.1273446083068848, 2.280330181121826, 2.184453010559082, 2.1324613094329834, 2.138012170791626]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3627, "test_loss": 1.8448671020507812}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3731, "test_loss": 1.8088125122070313}], "final": {"test_acc": 0.3731, "test_loss": 1.8088125122070313, "wall_s": 46.284276723861694, "cos_r_negg": [0.5217737555503845, 0.008525252342224121, -0.00947762094438076, 0.04014284908771515, 0.03533230721950531, 0.007415211759507656, 0.0013272948563098907, 0.060824908316135406, 0.021514687687158585, 0.023938609287142754, 0.07601390779018402, -0.061907485127449036, 0.04844936728477478, -0.023666687309741974, 0.040054649114608765, -0.03930802643299103, -0.021309133619070053, 0.01734125055372715, 0.015159038826823235, 0.06305719912052155, -0.05200711637735367, 0.03500454127788544, 0.031798359006643295, 0.042145244777202606, 0.08821828663349152, 0.04043489694595337, 0.04382440075278282, -0.004821183625608683, 0.013737156987190247, 0.0037198830395936966, 0.07937326282262802, 0.04212981089949608, -0.0645582526922226, 0.01643211394548416, 0.005947161465883255, 0.06351327896118164, 0.026483042165637016, 0.011229969561100006, -0.050125785171985626, 0.013528898358345032, 0.04318409413099289, 0.06815628707408905, -0.04041041433811188, 0.04399698227643967, -0.01782991737127304, -0.03676197677850723, 0.03483232110738754, 0.06307149678468704, -0.014615458436310291, -0.0490817129611969, -0.07918895781040192, -0.02049669437110424, -0.05999218672513962, 0.06805650144815445, 0.07911649346351624, 0.03073415346443653, 0.05748293176293373, 0.07361705601215363, 0.01881752908229828, 0.0017498719971626997], "cos_innovation_negg": [0.5217737555503845, 0.008525252342224121, -0.00947762094438076, 0.04014284908771515, 0.03533230721950531, 0.007415211759507656, 0.0013272948563098907, 0.060824908316135406, 0.021514687687158585, 0.023938609287142754, 0.07601390779018402, -0.061907485127449036, 0.04844936728477478, -0.023666687309741974, 0.040054649114608765, -0.03930802643299103, -0.021309133619070053, 0.01734125055372715, 0.015159038826823235, 0.06305719912052155, -0.05200711637735367, 0.03500454127788544, 0.031798359006643295, 0.042145244777202606, 0.08821828663349152, 0.04043489694595337, 0.04382440075278282, -0.004821183625608683, 0.013737156987190247, 0.0037198830395936966, 0.07937326282262802, 0.04212981089949608, -0.0645582526922226, 0.01643211394548416, 0.005947161465883255, 0.06351327896118164, 0.026483042165637016, 0.011229969561100006, -0.050125785171985626, 0.013528898358345032, 0.04318409413099289, 0.06815628707408905, -0.04041041433811188, 0.04399698227643967, -0.01782991737127304, -0.03676197677850723, 0.03483232110738754, 0.06307149678468704, -0.014615458436310291, -0.0490817129611969, -0.07918895781040192, -0.02049669437110424, -0.05999218672513962, 0.06805650144815445, 0.07911649346351624, 0.03073415346443653, 0.05748293176293373, 0.07361705601215363, 0.01881752908229828, 0.0017498719971626997], "cos_apical_negg": [0.5217737555503845, 0.008525252342224121, -0.00947762094438076, 0.04014284908771515, 0.03533230721950531, 0.007415211759507656, 0.0013272948563098907, 0.060824908316135406, 0.021514687687158585, 0.023938609287142754, 0.07601390779018402, -0.061907485127449036, 0.04844936728477478, -0.023666687309741974, 0.040054649114608765, -0.03930802643299103, -0.021309133619070053, 0.01734125055372715, 0.015159038826823235, 0.06305719912052155, -0.05200711637735367, 0.03500454127788544, 0.031798359006643295, 0.042145244777202606, 0.08821828663349152, 0.04043489694595337, 0.04382440075278282, -0.004821183625608683, 0.013737156987190247, 0.0037198830395936966, 0.07937326282262802, 0.04212981089949608, -0.0645582526922226, 0.01643211394548416, 0.005947161465883255, 0.06351327896118164, 0.026483042165637016, 0.011229969561100006, -0.050125785171985626, 0.013528898358345032, 0.04318409413099289, 0.06815628707408905, -0.04041041433811188, 0.04399698227643967, -0.01782991737127304, -0.03676197677850723, 0.03483232110738754, 0.06307149678468704, -0.014615458436310291, -0.0490817129611969, -0.07918895781040192, -0.02049669437110424, -0.05999218672513962, 0.06805650144815445, 0.07911649346351624, 0.03073415346443653, 0.05748293176293373, 0.07361705601215363, 0.01881752908229828, 0.0017498719971626997], "cos_Ac_negg": [0.5217737555503845, 0.008525252342224121, -0.00947762094438076, 0.04014284908771515, 0.03533230721950531, 0.007415211759507656, 0.0013272948563098907, 0.060824908316135406, 0.021514687687158585, 0.023938609287142754, 0.07601390779018402, -0.061907485127449036, 0.04844936728477478, -0.023666687309741974, 0.040054649114608765, -0.03930802643299103, -0.021309133619070053, 0.01734125055372715, 0.015159038826823235, 0.06305719912052155, -0.05200711637735367, 0.03500454127788544, 0.031798359006643295, 0.042145244777202606, 0.08821828663349152, 0.04043489694595337, 0.04382440075278282, -0.004821183625608683, 0.013737156987190247, 0.0037198830395936966, 0.07937326282262802, 0.04212981089949608, -0.0645582526922226, 0.01643211394548416, 0.005947161465883255, 0.06351327896118164, 0.026483042165637016, 0.011229969561100006, -0.050125785171985626, 0.013528898358345032, 0.04318409413099289, 0.06815628707408905, -0.04041041433811188, 0.04399698227643967, -0.01782991737127304, -0.03676197677850723, 0.03483232110738754, 0.06307149678468704, -0.014615458436310291, -0.0490817129611969, -0.07918895781040192, -0.02049669437110424, -0.05999218672513962, 0.06805650144815445, 0.07911649346351624, 0.03073415346443653, 0.05748293176293373, 0.07361705601215363, 0.01881752908229828, 0.0017498719971626997]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d60_s1.json b/results/scale_v3_cifar10_dfa_w64_d60_s1.json new file mode 100644 index 0000000..839e626 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d60_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d60_s1", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.473468780517578, "cos_r_negg": [-0.027598153799772263, -0.009432153776288033, 0.026431646198034286, 0.09308052062988281, 0.057336363941431046, 0.01887705735862255, 0.03829266130924225, 0.00904397014528513, -0.03249947726726532, -0.040997304022312164, 0.026174651458859444, -0.005292101763188839, -0.04484628885984421, -0.009162251837551594, 0.038208238780498505, -0.009770127013325691, -0.04561763256788254, 0.051819391548633575, -0.0008480320684611797, 0.003425545059144497, -0.03041260316967964, 0.0031225450802594423, 0.012182950973510742, -0.01790534518659115, -0.08202042430639267, 0.04760080203413963, -0.022590987384319305, -0.04574787616729736, -0.020178034901618958, 0.03437288478016853, -0.04592136666178703, 0.045652326196432114, -0.029149562120437622, 0.009575787000358105, 0.015170429833233356, 0.04743587598204613, -0.05227630212903023, 0.010222532786428928, 0.06895780563354492, -0.0300489142537117, -0.006607844494283199, 0.007750299759209156, 0.03843552619218826, -0.001963425427675247, -0.023244043812155724, 0.05353538691997528, -0.06197133660316467, -0.004330987110733986, 0.01083207968622446, -0.020775139331817627, -0.015730619430541992, 0.024743348360061646, 0.03138727694749832, -0.023993924260139465, 0.02352084219455719, -0.05231786146759987, 0.03779855743050575, -0.07508586347103119, 0.0621768981218338, -0.03642678260803223], "cos_innovation_negg": [-0.027598153799772263, -0.009432153776288033, 0.026431646198034286, 0.09308052062988281, 0.057336363941431046, 0.01887705735862255, 0.03829266130924225, 0.00904397014528513, -0.03249947726726532, -0.040997304022312164, 0.026174651458859444, -0.005292101763188839, -0.04484628885984421, -0.009162251837551594, 0.038208238780498505, -0.009770127013325691, -0.04561763256788254, 0.051819391548633575, -0.0008480320684611797, 0.003425545059144497, -0.03041260316967964, 0.0031225450802594423, 0.012182950973510742, -0.01790534518659115, -0.08202042430639267, 0.04760080203413963, -0.022590987384319305, -0.04574787616729736, -0.020178034901618958, 0.03437288478016853, -0.04592136666178703, 0.045652326196432114, -0.029149562120437622, 0.009575787000358105, 0.015170429833233356, 0.04743587598204613, -0.05227630212903023, 0.010222532786428928, 0.06895780563354492, -0.0300489142537117, -0.006607844494283199, 0.007750299759209156, 0.03843552619218826, -0.001963425427675247, -0.023244043812155724, 0.05353538691997528, -0.06197133660316467, -0.004330987110733986, 0.01083207968622446, -0.020775139331817627, -0.015730619430541992, 0.024743348360061646, 0.03138727694749832, -0.023993924260139465, 0.02352084219455719, -0.05231786146759987, 0.03779855743050575, -0.07508586347103119, 0.0621768981218338, -0.03642678260803223], "cos_apical_negg": [-0.027598153799772263, -0.009432153776288033, 0.026431646198034286, 0.09308052062988281, 0.057336363941431046, 0.01887705735862255, 0.03829266130924225, 0.00904397014528513, -0.03249947726726532, -0.040997304022312164, 0.026174651458859444, -0.005292101763188839, -0.04484628885984421, -0.009162251837551594, 0.038208238780498505, -0.009770127013325691, -0.04561763256788254, 0.051819391548633575, -0.0008480320684611797, 0.003425545059144497, -0.03041260316967964, 0.0031225450802594423, 0.012182950973510742, -0.01790534518659115, -0.08202042430639267, 0.04760080203413963, -0.022590987384319305, -0.04574787616729736, -0.020178034901618958, 0.03437288478016853, -0.04592136666178703, 0.045652326196432114, -0.029149562120437622, 0.009575787000358105, 0.015170429833233356, 0.04743587598204613, -0.05227630212903023, 0.010222532786428928, 0.06895780563354492, -0.0300489142537117, -0.006607844494283199, 0.007750299759209156, 0.03843552619218826, -0.001963425427675247, -0.023244043812155724, 0.05353538691997528, -0.06197133660316467, -0.004330987110733986, 0.01083207968622446, -0.020775139331817627, -0.015730619430541992, 0.024743348360061646, 0.03138727694749832, -0.023993924260139465, 0.02352084219455719, -0.05231786146759987, 0.03779855743050575, -0.07508586347103119, 0.0621768981218338, -0.03642678260803223], "cos_Ac_negg": [-0.027598153799772263, -0.009432153776288033, 0.026431646198034286, 0.09308052062988281, 0.057336363941431046, 0.01887705735862255, 0.03829266130924225, 0.00904397014528513, -0.03249947726726532, -0.040997304022312164, 0.026174651458859444, -0.005292101763188839, -0.04484628885984421, -0.009162251837551594, 0.038208238780498505, -0.009770127013325691, -0.04561763256788254, 0.051819391548633575, -0.0008480320684611797, 0.003425545059144497, -0.03041260316967964, 0.0031225450802594423, 0.012182950973510742, -0.01790534518659115, -0.08202042430639267, 0.04760080203413963, -0.022590987384319305, -0.04574787616729736, -0.020178034901618958, 0.03437288478016853, -0.04592136666178703, 0.045652326196432114, -0.029149562120437622, 0.009575787000358105, 0.015170429833233356, 0.04743587598204613, -0.05227630212903023, 0.010222532786428928, 0.06895780563354492, -0.0300489142537117, -0.006607844494283199, 0.007750299759209156, 0.03843552619218826, -0.001963425427675247, -0.023244043812155724, 0.05353538691997528, -0.06197133660316467, -0.004330987110733986, 0.01083207968622446, -0.020775139331817627, -0.015730619430541992, 0.024743348360061646, 0.03138727694749832, -0.023993924260139465, 0.02352084219455719, -0.05231786146759987, 0.03779855743050575, -0.07508586347103119, 0.0621768981218338, -0.03642678260803223], "r_norm": [2.4270968437194824, 2.434605121612549, 2.3561136722564697, 2.5349457263946533, 2.2505202293395996, 2.491443634033203, 2.501532554626465, 2.3465404510498047, 2.49263858795166, 2.4819836616516113, 2.39988374710083, 2.449678421020508, 2.480316162109375, 2.369499683380127, 2.446897506713867, 2.3385167121887207, 2.4765162467956543, 2.481140375137329, 2.4799866676330566, 2.5715649127960205, 2.510502338409424, 2.4862959384918213, 2.426884174346924, 2.469876527786255, 2.4370555877685547, 2.372319459915161, 2.422351598739624, 2.407158613204956, 2.4810409545898438, 2.434788703918457, 2.362978458404541, 2.5303244590759277, 2.4963393211364746, 2.398350715637207, 2.532294511795044, 2.3016581535339355, 2.4103474617004395, 2.3904144763946533, 2.5655782222747803, 2.247401714324951, 2.4511404037475586, 2.6094412803649902, 2.458674430847168, 2.452883243560791, 2.5604450702667236, 2.404538154602051, 2.4014854431152344, 2.3303277492523193, 2.4563159942626953, 2.418234348297119, 2.4149742126464844, 2.5191779136657715, 2.416952610015869, 2.4506287574768066, 2.4003663063049316, 2.403989553451538, 2.5814437866210938, 2.4009132385253906, 2.4801764488220215, 2.5884768962860107]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3451, "test_loss": 1.8796652954101563}, {"step": 500, "epoch": 1, "train_loss": 1.7371916770935059, "cos_r_negg": [0.49210307002067566, -0.021473165601491928, -0.019318562000989914, -0.034488923847675323, -0.016506116837263107, -0.032033126801252365, 0.0025502550415694714, -0.0022578458301723003, 0.0224224254488945, 0.10033118724822998, -0.02180098183453083, -0.01039479486644268, 0.06380053609609604, 0.05439285933971405, 0.05308428779244423, -0.050040699541568756, -0.0016513345763087273, 0.10445742309093475, 0.060675397515296936, 0.05194630101323128, 0.024879835546016693, 0.02086039073765278, -0.0091919656842947, 0.003746760543435812, -0.009052399545907974, -0.015374921262264252, 0.015182260423898697, -0.0510367825627327, -0.07887183129787445, 0.03440482169389725, 0.025525961071252823, -0.048653438687324524, -0.03527345508337021, 0.004242987837642431, -0.0021380733232945204, -0.04661485552787781, 0.024683985859155655, -0.10638971626758575, 0.031551167368888855, 0.020364129915833473, 0.04044719040393829, 0.030624084174633026, 0.0031502838246524334, 0.00035383040085434914, -0.028015904128551483, -0.0009205099195241928, -0.01525571197271347, -0.038573652505874634, 0.038986485451459885, -0.06565085053443909, 0.017354248091578484, 0.03425765782594681, 0.05934640020132065, 0.018150832504034042, -0.017445560544729233, -0.04361693561077118, -0.013200575485825539, -0.06447014957666397, 0.009411916136741638, -0.00677395099774003], "cos_innovation_negg": [0.49210307002067566, -0.021473165601491928, -0.019318562000989914, -0.034488923847675323, -0.016506116837263107, -0.032033126801252365, 0.0025502550415694714, -0.0022578458301723003, 0.0224224254488945, 0.10033118724822998, -0.02180098183453083, -0.01039479486644268, 0.06380053609609604, 0.05439285933971405, 0.05308428779244423, -0.050040699541568756, -0.0016513345763087273, 0.10445742309093475, 0.060675397515296936, 0.05194630101323128, 0.024879835546016693, 0.02086039073765278, -0.0091919656842947, 0.003746760543435812, -0.009052399545907974, -0.015374921262264252, 0.015182260423898697, -0.0510367825627327, -0.07887183129787445, 0.03440482169389725, 0.025525961071252823, -0.048653438687324524, -0.03527345508337021, 0.004242987837642431, -0.0021380733232945204, -0.04661485552787781, 0.024683985859155655, -0.10638971626758575, 0.031551167368888855, 0.020364129915833473, 0.04044719040393829, 0.030624084174633026, 0.0031502838246524334, 0.00035383040085434914, -0.028015904128551483, -0.0009205099195241928, -0.01525571197271347, -0.038573652505874634, 0.038986485451459885, -0.06565085053443909, 0.017354248091578484, 0.03425765782594681, 0.05934640020132065, 0.018150832504034042, -0.017445560544729233, -0.04361693561077118, -0.013200575485825539, -0.06447014957666397, 0.009411916136741638, -0.00677395099774003], "cos_apical_negg": [0.49210307002067566, -0.021473165601491928, -0.019318562000989914, -0.034488923847675323, -0.016506116837263107, -0.032033126801252365, 0.0025502550415694714, -0.0022578458301723003, 0.0224224254488945, 0.10033118724822998, -0.02180098183453083, -0.01039479486644268, 0.06380053609609604, 0.05439285933971405, 0.05308428779244423, -0.050040699541568756, -0.0016513345763087273, 0.10445742309093475, 0.060675397515296936, 0.05194630101323128, 0.024879835546016693, 0.02086039073765278, -0.0091919656842947, 0.003746760543435812, -0.009052399545907974, -0.015374921262264252, 0.015182260423898697, -0.0510367825627327, -0.07887183129787445, 0.03440482169389725, 0.025525961071252823, -0.048653438687324524, -0.03527345508337021, 0.004242987837642431, -0.0021380733232945204, -0.04661485552787781, 0.024683985859155655, -0.10638971626758575, 0.031551167368888855, 0.020364129915833473, 0.04044719040393829, 0.030624084174633026, 0.0031502838246524334, 0.00035383040085434914, -0.028015904128551483, -0.0009205099195241928, -0.01525571197271347, -0.038573652505874634, 0.038986485451459885, -0.06565085053443909, 0.017354248091578484, 0.03425765782594681, 0.05934640020132065, 0.018150832504034042, -0.017445560544729233, -0.04361693561077118, -0.013200575485825539, -0.06447014957666397, 0.009411916136741638, -0.00677395099774003], "cos_Ac_negg": [0.49210307002067566, -0.021473165601491928, -0.019318562000989914, -0.034488923847675323, -0.016506116837263107, -0.032033126801252365, 0.0025502550415694714, -0.0022578458301723003, 0.0224224254488945, 0.10033118724822998, -0.02180098183453083, -0.01039479486644268, 0.06380053609609604, 0.05439285933971405, 0.05308428779244423, -0.050040699541568756, -0.0016513345763087273, 0.10445742309093475, 0.060675397515296936, 0.05194630101323128, 0.024879835546016693, 0.02086039073765278, -0.0091919656842947, 0.003746760543435812, -0.009052399545907974, -0.015374921262264252, 0.015182260423898697, -0.0510367825627327, -0.07887183129787445, 0.03440482169389725, 0.025525961071252823, -0.048653438687324524, -0.03527345508337021, 0.004242987837642431, -0.0021380733232945204, -0.04661485552787781, 0.024683985859155655, -0.10638971626758575, 0.031551167368888855, 0.020364129915833473, 0.04044719040393829, 0.030624084174633026, 0.0031502838246524334, 0.00035383040085434914, -0.028015904128551483, -0.0009205099195241928, -0.01525571197271347, -0.038573652505874634, 0.038986485451459885, -0.06565085053443909, 0.017354248091578484, 0.03425765782594681, 0.05934640020132065, 0.018150832504034042, -0.017445560544729233, -0.04361693561077118, -0.013200575485825539, -0.06447014957666397, 0.009411916136741638, -0.00677395099774003], "r_norm": [2.1445868015289307, 2.1550099849700928, 2.0823798179626465, 2.257019281387329, 2.006627082824707, 2.2138407230377197, 2.2126355171203613, 2.0646674633026123, 2.193983793258667, 2.2017979621887207, 2.114409923553467, 2.166790008544922, 2.206501007080078, 2.1183559894561768, 2.1682443618774414, 2.078402042388916, 2.181321620941162, 2.216914653778076, 2.179603099822998, 2.2880125045776367, 2.215944766998291, 2.206589460372925, 2.1444778442382812, 2.1836419105529785, 2.1342477798461914, 2.10422420501709, 2.1536765098571777, 2.118281841278076, 2.1893062591552734, 2.1553542613983154, 2.0882463455200195, 2.2499961853027344, 2.1941771507263184, 2.123349666595459, 2.237406015396118, 2.0561983585357666, 2.1399145126342773, 2.0994865894317627, 2.281294107437134, 1.986048698425293, 2.1779251098632812, 2.328061580657959, 2.17801570892334, 2.1552720069885254, 2.282252550125122, 2.1189169883728027, 2.1303539276123047, 2.0720603466033936, 2.1853036880493164, 2.1267967224121094, 2.1461400985717773, 2.219489574432373, 2.1441023349761963, 2.171125888824463, 2.127178907394409, 2.1242194175720215, 2.287989616394043, 2.1422922611236572, 2.1876091957092285, 2.2962634563446045]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3675, "test_loss": 1.8437145263671875}, {"step": 1000, "epoch": 2, "train_loss": 1.856551170349121, "cos_r_negg": [0.5896940231323242, -0.016529276967048645, -0.029532531276345253, -0.020580170676112175, 0.009227761998772621, 0.04167924076318741, -0.027592528611421585, -0.0019916342571377754, 0.02364553138613701, 0.008891643956303596, 0.03972601890563965, -0.02918899431824684, 0.06632271409034729, 0.050521206110715866, -0.016979701817035675, -0.010578258894383907, 0.0038482481613755226, 0.12065346539020538, 0.008864634670317173, 0.06177801638841629, 0.0210802610963583, 0.03776024654507637, -0.03447148948907852, -0.02465098723769188, -0.01338434312492609, -0.0014518045354634523, -0.0058770873583853245, -0.05886879563331604, -0.039693381637334824, 0.044943518936634064, 0.011715270578861237, 0.025624440982937813, -0.023362180218100548, -0.02046864666044712, 0.026639800518751144, 0.0292205773293972, -0.02216409519314766, -0.08396327495574951, 0.08962459117174149, -0.003321272786706686, -0.059627629816532135, 0.03026420809328556, 0.01427528541535139, -0.09789499640464783, 0.016419434919953346, 0.008588830009102821, 0.04360141232609749, 0.0004934566095471382, 0.055750422179698944, 0.027535904198884964, 0.02505086176097393, 0.017078274860978127, 0.03557826578617096, -0.002249807817861438, -0.010592024773359299, -0.04078730195760727, -0.021078377962112427, -0.07171814143657684, -0.005808907561004162, -0.014510632492601871], "cos_innovation_negg": [0.5896940231323242, -0.016529276967048645, -0.029532531276345253, -0.020580170676112175, 0.009227761998772621, 0.04167924076318741, -0.027592528611421585, -0.0019916342571377754, 0.02364553138613701, 0.008891643956303596, 0.03972601890563965, -0.02918899431824684, 0.06632271409034729, 0.050521206110715866, -0.016979701817035675, -0.010578258894383907, 0.0038482481613755226, 0.12065346539020538, 0.008864634670317173, 0.06177801638841629, 0.0210802610963583, 0.03776024654507637, -0.03447148948907852, -0.02465098723769188, -0.01338434312492609, -0.0014518045354634523, -0.0058770873583853245, -0.05886879563331604, -0.039693381637334824, 0.044943518936634064, 0.011715270578861237, 0.025624440982937813, -0.023362180218100548, -0.02046864666044712, 0.026639800518751144, 0.0292205773293972, -0.02216409519314766, -0.08396327495574951, 0.08962459117174149, -0.003321272786706686, -0.059627629816532135, 0.03026420809328556, 0.01427528541535139, -0.09789499640464783, 0.016419434919953346, 0.008588830009102821, 0.04360141232609749, 0.0004934566095471382, 0.055750422179698944, 0.027535904198884964, 0.02505086176097393, 0.017078274860978127, 0.03557826578617096, -0.002249807817861438, -0.010592024773359299, -0.04078730195760727, -0.021078377962112427, -0.07171814143657684, -0.005808907561004162, -0.014510632492601871], "cos_apical_negg": [0.5896940231323242, -0.016529276967048645, -0.029532531276345253, -0.020580170676112175, 0.009227761998772621, 0.04167924076318741, -0.027592528611421585, -0.0019916342571377754, 0.02364553138613701, 0.008891643956303596, 0.03972601890563965, -0.02918899431824684, 0.06632271409034729, 0.050521206110715866, -0.016979701817035675, -0.010578258894383907, 0.0038482481613755226, 0.12065346539020538, 0.008864634670317173, 0.06177801638841629, 0.0210802610963583, 0.03776024654507637, -0.03447148948907852, -0.02465098723769188, -0.01338434312492609, -0.0014518045354634523, -0.0058770873583853245, -0.05886879563331604, -0.039693381637334824, 0.044943518936634064, 0.011715270578861237, 0.025624440982937813, -0.023362180218100548, -0.02046864666044712, 0.026639800518751144, 0.0292205773293972, -0.02216409519314766, -0.08396327495574951, 0.08962459117174149, -0.003321272786706686, -0.059627629816532135, 0.03026420809328556, 0.01427528541535139, -0.09789499640464783, 0.016419434919953346, 0.008588830009102821, 0.04360141232609749, 0.0004934566095471382, 0.055750422179698944, 0.027535904198884964, 0.02505086176097393, 0.017078274860978127, 0.03557826578617096, -0.002249807817861438, -0.010592024773359299, -0.04078730195760727, -0.021078377962112427, -0.07171814143657684, -0.005808907561004162, -0.014510632492601871], "cos_Ac_negg": [0.5896940231323242, -0.016529276967048645, -0.029532531276345253, -0.020580170676112175, 0.009227761998772621, 0.04167924076318741, -0.027592528611421585, -0.0019916342571377754, 0.02364553138613701, 0.008891643956303596, 0.03972601890563965, -0.02918899431824684, 0.06632271409034729, 0.050521206110715866, -0.016979701817035675, -0.010578258894383907, 0.0038482481613755226, 0.12065346539020538, 0.008864634670317173, 0.06177801638841629, 0.0210802610963583, 0.03776024654507637, -0.03447148948907852, -0.02465098723769188, -0.01338434312492609, -0.0014518045354634523, -0.0058770873583853245, -0.05886879563331604, -0.039693381637334824, 0.044943518936634064, 0.011715270578861237, 0.025624440982937813, -0.023362180218100548, -0.02046864666044712, 0.026639800518751144, 0.0292205773293972, -0.02216409519314766, -0.08396327495574951, 0.08962459117174149, -0.003321272786706686, -0.059627629816532135, 0.03026420809328556, 0.01427528541535139, -0.09789499640464783, 0.016419434919953346, 0.008588830009102821, 0.04360141232609749, 0.0004934566095471382, 0.055750422179698944, 0.027535904198884964, 0.02505086176097393, 0.017078274860978127, 0.03557826578617096, -0.002249807817861438, -0.010592024773359299, -0.04078730195760727, -0.021078377962112427, -0.07171814143657684, -0.005808907561004162, -0.014510632492601871], "r_norm": [2.1279706954956055, 2.1452014446258545, 2.0762834548950195, 2.248734474182129, 2.0010569095611572, 2.2004799842834473, 2.1894655227661133, 2.0548644065856934, 2.186929225921631, 2.1845827102661133, 2.10275936126709, 2.1651458740234375, 2.178525686264038, 2.1044094562530518, 2.1501781940460205, 2.0781033039093018, 2.1717092990875244, 2.1952908039093018, 2.1651721000671387, 2.2762255668640137, 2.1976256370544434, 2.1923625469207764, 2.146271228790283, 2.1650543212890625, 2.135721206665039, 2.084906578063965, 2.1426587104797363, 2.1156680583953857, 2.182443141937256, 2.1459617614746094, 2.0700299739837646, 2.236079692840576, 2.183598518371582, 2.1092684268951416, 2.2199578285217285, 2.0424752235412598, 2.142794370651245, 2.08893084526062, 2.2793500423431396, 1.9716095924377441, 2.1676828861236572, 2.3256146907806396, 2.1735785007476807, 2.153728485107422, 2.2679905891418457, 2.113356351852417, 2.1217899322509766, 2.065080165863037, 2.1765012741088867, 2.1130459308624268, 2.140854597091675, 2.206801176071167, 2.1323294639587402, 2.1515865325927734, 2.124094009399414, 2.1142683029174805, 2.284008026123047, 2.125511884689331, 2.186213254928589, 2.2831428050994873]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3461, "test_loss": 1.8577670166015625}, {"step": 1500, "epoch": 3, "train_loss": 1.740715503692627, "cos_r_negg": [0.5852367877960205, 0.000259526539593935, 0.04429687559604645, -0.02264067716896534, -0.020473238080739975, 0.04130129516124725, -0.008357562124729156, 0.025883030146360397, -0.008155830204486847, 0.019729599356651306, 0.0815841406583786, 0.02089286409318447, 0.033344633877277374, -0.02094246819615364, -0.012538651935756207, -0.007486504502594471, 0.020026618614792824, 0.09310787171125412, 0.02987789735198021, 0.012750264257192612, 0.03524679318070412, -0.0033787558786571026, -0.04740176349878311, -0.0011798343621194363, -0.06386824697256088, 0.009064853191375732, 0.05379222333431244, -0.050802573561668396, -0.023035673424601555, 0.06920269131660461, 0.006249513477087021, 0.013661675155162811, 0.017296405509114265, -0.032514479011297226, -0.0006749979220330715, -0.007619215175509453, 0.024516776204109192, -0.10596607625484467, 0.029622860252857208, -0.009037001058459282, -0.09730689227581024, 0.015918467193841934, 0.026134124025702477, -0.12272250652313232, -0.03816564008593559, -0.03450819104909897, 0.03738105669617653, -0.0032423329539597034, 0.023222865536808968, 0.03185231611132622, 0.03808435797691345, 0.007363161072134972, 0.08140464872121811, 0.041655249893665314, 0.028157301247119904, -0.02042519673705101, -0.024911191314458847, -0.07341480255126953, 0.0178495142608881, -0.011052543297410011], "cos_innovation_negg": [0.5852367877960205, 0.000259526539593935, 0.04429687559604645, -0.02264067716896534, -0.020473238080739975, 0.04130129516124725, -0.008357562124729156, 0.025883030146360397, -0.008155830204486847, 0.019729599356651306, 0.0815841406583786, 0.02089286409318447, 0.033344633877277374, -0.02094246819615364, -0.012538651935756207, -0.007486504502594471, 0.020026618614792824, 0.09310787171125412, 0.02987789735198021, 0.012750264257192612, 0.03524679318070412, -0.0033787558786571026, -0.04740176349878311, -0.0011798343621194363, -0.06386824697256088, 0.009064853191375732, 0.05379222333431244, -0.050802573561668396, -0.023035673424601555, 0.06920269131660461, 0.006249513477087021, 0.013661675155162811, 0.017296405509114265, -0.032514479011297226, -0.0006749979220330715, -0.007619215175509453, 0.024516776204109192, -0.10596607625484467, 0.029622860252857208, -0.009037001058459282, -0.09730689227581024, 0.015918467193841934, 0.026134124025702477, -0.12272250652313232, -0.03816564008593559, -0.03450819104909897, 0.03738105669617653, -0.0032423329539597034, 0.023222865536808968, 0.03185231611132622, 0.03808435797691345, 0.007363161072134972, 0.08140464872121811, 0.041655249893665314, 0.028157301247119904, -0.02042519673705101, -0.024911191314458847, -0.07341480255126953, 0.0178495142608881, -0.011052543297410011], "cos_apical_negg": [0.5852367877960205, 0.000259526539593935, 0.04429687559604645, -0.02264067716896534, -0.020473238080739975, 0.04130129516124725, -0.008357562124729156, 0.025883030146360397, -0.008155830204486847, 0.019729599356651306, 0.0815841406583786, 0.02089286409318447, 0.033344633877277374, -0.02094246819615364, -0.012538651935756207, -0.007486504502594471, 0.020026618614792824, 0.09310787171125412, 0.02987789735198021, 0.012750264257192612, 0.03524679318070412, -0.0033787558786571026, -0.04740176349878311, -0.0011798343621194363, -0.06386824697256088, 0.009064853191375732, 0.05379222333431244, -0.050802573561668396, -0.023035673424601555, 0.06920269131660461, 0.006249513477087021, 0.013661675155162811, 0.017296405509114265, -0.032514479011297226, -0.0006749979220330715, -0.007619215175509453, 0.024516776204109192, -0.10596607625484467, 0.029622860252857208, -0.009037001058459282, -0.09730689227581024, 0.015918467193841934, 0.026134124025702477, -0.12272250652313232, -0.03816564008593559, -0.03450819104909897, 0.03738105669617653, -0.0032423329539597034, 0.023222865536808968, 0.03185231611132622, 0.03808435797691345, 0.007363161072134972, 0.08140464872121811, 0.041655249893665314, 0.028157301247119904, -0.02042519673705101, -0.024911191314458847, -0.07341480255126953, 0.0178495142608881, -0.011052543297410011], "cos_Ac_negg": [0.5852367877960205, 0.000259526539593935, 0.04429687559604645, -0.02264067716896534, -0.020473238080739975, 0.04130129516124725, -0.008357562124729156, 0.025883030146360397, -0.008155830204486847, 0.019729599356651306, 0.0815841406583786, 0.02089286409318447, 0.033344633877277374, -0.02094246819615364, -0.012538651935756207, -0.007486504502594471, 0.020026618614792824, 0.09310787171125412, 0.02987789735198021, 0.012750264257192612, 0.03524679318070412, -0.0033787558786571026, -0.04740176349878311, -0.0011798343621194363, -0.06386824697256088, 0.009064853191375732, 0.05379222333431244, -0.050802573561668396, -0.023035673424601555, 0.06920269131660461, 0.006249513477087021, 0.013661675155162811, 0.017296405509114265, -0.032514479011297226, -0.0006749979220330715, -0.007619215175509453, 0.024516776204109192, -0.10596607625484467, 0.029622860252857208, -0.009037001058459282, -0.09730689227581024, 0.015918467193841934, 0.026134124025702477, -0.12272250652313232, -0.03816564008593559, -0.03450819104909897, 0.03738105669617653, -0.0032423329539597034, 0.023222865536808968, 0.03185231611132622, 0.03808435797691345, 0.007363161072134972, 0.08140464872121811, 0.041655249893665314, 0.028157301247119904, -0.02042519673705101, -0.024911191314458847, -0.07341480255126953, 0.0178495142608881, -0.011052543297410011], "r_norm": [2.148435592651367, 2.1588828563690186, 2.0891246795654297, 2.271038055419922, 2.0216116905212402, 2.2181501388549805, 2.2066891193389893, 2.068427085876465, 2.1909337043762207, 2.2031679153442383, 2.122901439666748, 2.1715993881225586, 2.1984009742736816, 2.122238874435425, 2.166700839996338, 2.088395118713379, 2.1877400875091553, 2.210538387298584, 2.185739040374756, 2.297398090362549, 2.220534563064575, 2.2155814170837402, 2.1514792442321777, 2.185685634613037, 2.1353836059570312, 2.105055093765259, 2.159041404724121, 2.130660057067871, 2.1972804069519043, 2.1660313606262207, 2.0949435234069824, 2.251980781555176, 2.206101417541504, 2.1316826343536377, 2.2430129051208496, 2.0598392486572266, 2.149430274963379, 2.099935293197632, 2.2979445457458496, 1.979936122894287, 2.181729316711426, 2.339931011199951, 2.181119918823242, 2.171027421951294, 2.284125804901123, 2.123413562774658, 2.138547658920288, 2.071990489959717, 2.186458110809326, 2.1363978385925293, 2.153261184692383, 2.225515127182007, 2.1529626846313477, 2.1665096282958984, 2.1399221420288086, 2.13191819190979, 2.2970333099365234, 2.1474978923797607, 2.197443962097168, 2.307278871536255]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3721, "test_loss": 1.812070849609375}, {"epoch_end": 4, "step": 1955, "test_acc": 0.364, "test_loss": 1.8201409912109374}], "final": {"test_acc": 0.364, "test_loss": 1.8201409912109374, "wall_s": 44.53204822540283, "cos_r_negg": [0.5663803219795227, 0.02666696533560753, -0.01109970360994339, 0.0219368077814579, -0.029507845640182495, 0.05618344992399216, 0.02701859548687935, 0.04832802712917328, 0.015132657252252102, 0.023101039230823517, 0.057635799050331116, 0.009037761017680168, 0.020659800618886948, 0.004687355365604162, 0.004771437495946884, -0.06894863396883011, 0.03974350541830063, 0.15389308333396912, 0.031142299994826317, 0.013322555460035801, 0.03790561854839325, 0.04704427346587181, -0.030945628881454468, 0.0577218160033226, -0.0007702326402068138, -0.0003890441730618477, 0.05555436760187149, -0.01514861173927784, -0.017457041889429092, 0.03646022081375122, 0.004875186365097761, -0.013903830200433731, 0.009979970753192902, 0.027549315243959427, 0.013523411005735397, 0.01834261417388916, 0.010299532674252987, -0.02055676095187664, 0.0901731625199318, 0.014240654185414314, -0.03428094834089279, -0.007334109395742416, 0.023133091628551483, -0.06200602650642395, 0.007969506084918976, 0.011513827368617058, 0.014754770323634148, 0.003776920959353447, 0.028728565201163292, 0.05010197311639786, 0.005072337109595537, 0.029661886394023895, 0.06985858082771301, 0.01944391056895256, 0.006242034025490284, 0.014598161913454533, -0.003918610978871584, -0.06260689347982407, 0.01665465347468853, 0.02490917593240738], "cos_innovation_negg": [0.5663803219795227, 0.02666696533560753, -0.01109970360994339, 0.0219368077814579, -0.029507845640182495, 0.05618344992399216, 0.02701859548687935, 0.04832802712917328, 0.015132657252252102, 0.023101039230823517, 0.057635799050331116, 0.009037761017680168, 0.020659800618886948, 0.004687355365604162, 0.004771437495946884, -0.06894863396883011, 0.03974350541830063, 0.15389308333396912, 0.031142299994826317, 0.013322555460035801, 0.03790561854839325, 0.04704427346587181, -0.030945628881454468, 0.0577218160033226, -0.0007702326402068138, -0.0003890441730618477, 0.05555436760187149, -0.01514861173927784, -0.017457041889429092, 0.03646022081375122, 0.004875186365097761, -0.013903830200433731, 0.009979970753192902, 0.027549315243959427, 0.013523411005735397, 0.01834261417388916, 0.010299532674252987, -0.02055676095187664, 0.0901731625199318, 0.014240654185414314, -0.03428094834089279, -0.007334109395742416, 0.023133091628551483, -0.06200602650642395, 0.007969506084918976, 0.011513827368617058, 0.014754770323634148, 0.003776920959353447, 0.028728565201163292, 0.05010197311639786, 0.005072337109595537, 0.029661886394023895, 0.06985858082771301, 0.01944391056895256, 0.006242034025490284, 0.014598161913454533, -0.003918610978871584, -0.06260689347982407, 0.01665465347468853, 0.02490917593240738], "cos_apical_negg": [0.5663803219795227, 0.02666696533560753, -0.01109970360994339, 0.0219368077814579, -0.029507845640182495, 0.05618344992399216, 0.02701859548687935, 0.04832802712917328, 0.015132657252252102, 0.023101039230823517, 0.057635799050331116, 0.009037761017680168, 0.020659800618886948, 0.004687355365604162, 0.004771437495946884, -0.06894863396883011, 0.03974350541830063, 0.15389308333396912, 0.031142299994826317, 0.013322555460035801, 0.03790561854839325, 0.04704427346587181, -0.030945628881454468, 0.0577218160033226, -0.0007702326402068138, -0.0003890441730618477, 0.05555436760187149, -0.01514861173927784, -0.017457041889429092, 0.03646022081375122, 0.004875186365097761, -0.013903830200433731, 0.009979970753192902, 0.027549315243959427, 0.013523411005735397, 0.01834261417388916, 0.010299532674252987, -0.02055676095187664, 0.0901731625199318, 0.014240654185414314, -0.03428094834089279, -0.007334109395742416, 0.023133091628551483, -0.06200602650642395, 0.007969506084918976, 0.011513827368617058, 0.014754770323634148, 0.003776920959353447, 0.028728565201163292, 0.05010197311639786, 0.005072337109595537, 0.029661886394023895, 0.06985858082771301, 0.01944391056895256, 0.006242034025490284, 0.014598161913454533, -0.003918610978871584, -0.06260689347982407, 0.01665465347468853, 0.02490917593240738], "cos_Ac_negg": [0.5663803219795227, 0.02666696533560753, -0.01109970360994339, 0.0219368077814579, -0.029507845640182495, 0.05618344992399216, 0.02701859548687935, 0.04832802712917328, 0.015132657252252102, 0.023101039230823517, 0.057635799050331116, 0.009037761017680168, 0.020659800618886948, 0.004687355365604162, 0.004771437495946884, -0.06894863396883011, 0.03974350541830063, 0.15389308333396912, 0.031142299994826317, 0.013322555460035801, 0.03790561854839325, 0.04704427346587181, -0.030945628881454468, 0.0577218160033226, -0.0007702326402068138, -0.0003890441730618477, 0.05555436760187149, -0.01514861173927784, -0.017457041889429092, 0.03646022081375122, 0.004875186365097761, -0.013903830200433731, 0.009979970753192902, 0.027549315243959427, 0.013523411005735397, 0.01834261417388916, 0.010299532674252987, -0.02055676095187664, 0.0901731625199318, 0.014240654185414314, -0.03428094834089279, -0.007334109395742416, 0.023133091628551483, -0.06200602650642395, 0.007969506084918976, 0.011513827368617058, 0.014754770323634148, 0.003776920959353447, 0.028728565201163292, 0.05010197311639786, 0.005072337109595537, 0.029661886394023895, 0.06985858082771301, 0.01944391056895256, 0.006242034025490284, 0.014598161913454533, -0.003918610978871584, -0.06260689347982407, 0.01665465347468853, 0.02490917593240738]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d60_s2.json b/results/scale_v3_cifar10_dfa_w64_d60_s2.json new file mode 100644 index 0000000..55a46ef --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d60_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d60_s2", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.456728219985962, "cos_r_negg": [0.10487774759531021, 0.017078710719943047, -0.013036027550697327, -0.0992920845746994, -0.003689222037792206, 0.034252963960170746, -0.04259507730603218, -0.02045252174139023, -0.045132823288440704, 0.025059916079044342, 0.08817455917596817, 0.010035283863544464, 0.06160152703523636, 0.014218760654330254, -0.00823059119284153, -0.06906627863645554, 0.004505272023379803, 0.026955002918839455, -0.03240483999252319, 0.022147294133901596, -0.018964610993862152, 0.050118640065193176, -0.04241577535867691, -0.03627433627843857, -0.01068196166306734, -0.0005757552571594715, -0.07351330667734146, 0.0071894340217113495, -0.0036361757665872574, -0.04875703901052475, 0.06124519556760788, 0.041685089468955994, 0.02372598648071289, -0.06642483919858932, -0.07185755670070648, -0.034029535949230194, -0.04177357256412506, -0.03938595950603485, 0.02855924889445305, -0.011848047375679016, -0.016205070540308952, 0.043828852474689484, 0.024598635733127594, -0.024466268718242645, -0.044855717569589615, 0.057504862546920776, 0.04564825817942619, -0.010068122297525406, 0.08641175180673599, 0.12965255975723267, 0.05013319104909897, 0.01800365559756756, 0.07069613039493561, -0.028849728405475616, -0.04724215343594551, -0.0934894010424614, -0.1059950739145279, 0.0669117122888565, 0.012815272435545921, 0.022690586745738983], "cos_innovation_negg": [0.10487774759531021, 0.017078710719943047, -0.013036027550697327, -0.0992920845746994, -0.003689222037792206, 0.034252963960170746, -0.04259507730603218, -0.02045252174139023, -0.045132823288440704, 0.025059916079044342, 0.08817455917596817, 0.010035283863544464, 0.06160152703523636, 0.014218760654330254, -0.00823059119284153, -0.06906627863645554, 0.004505272023379803, 0.026955002918839455, -0.03240483999252319, 0.022147294133901596, -0.018964610993862152, 0.050118640065193176, -0.04241577535867691, -0.03627433627843857, -0.01068196166306734, -0.0005757552571594715, -0.07351330667734146, 0.0071894340217113495, -0.0036361757665872574, -0.04875703901052475, 0.06124519556760788, 0.041685089468955994, 0.02372598648071289, -0.06642483919858932, -0.07185755670070648, -0.034029535949230194, -0.04177357256412506, -0.03938595950603485, 0.02855924889445305, -0.011848047375679016, -0.016205070540308952, 0.043828852474689484, 0.024598635733127594, -0.024466268718242645, -0.044855717569589615, 0.057504862546920776, 0.04564825817942619, -0.010068122297525406, 0.08641175180673599, 0.12965255975723267, 0.05013319104909897, 0.01800365559756756, 0.07069613039493561, -0.028849728405475616, -0.04724215343594551, -0.0934894010424614, -0.1059950739145279, 0.0669117122888565, 0.012815272435545921, 0.022690586745738983], "cos_apical_negg": [0.10487774759531021, 0.017078710719943047, -0.013036027550697327, -0.0992920845746994, -0.003689222037792206, 0.034252963960170746, -0.04259507730603218, -0.02045252174139023, -0.045132823288440704, 0.025059916079044342, 0.08817455917596817, 0.010035283863544464, 0.06160152703523636, 0.014218760654330254, -0.00823059119284153, -0.06906627863645554, 0.004505272023379803, 0.026955002918839455, -0.03240483999252319, 0.022147294133901596, -0.018964610993862152, 0.050118640065193176, -0.04241577535867691, -0.03627433627843857, -0.01068196166306734, -0.0005757552571594715, -0.07351330667734146, 0.0071894340217113495, -0.0036361757665872574, -0.04875703901052475, 0.06124519556760788, 0.041685089468955994, 0.02372598648071289, -0.06642483919858932, -0.07185755670070648, -0.034029535949230194, -0.04177357256412506, -0.03938595950603485, 0.02855924889445305, -0.011848047375679016, -0.016205070540308952, 0.043828852474689484, 0.024598635733127594, -0.024466268718242645, -0.044855717569589615, 0.057504862546920776, 0.04564825817942619, -0.010068122297525406, 0.08641175180673599, 0.12965255975723267, 0.05013319104909897, 0.01800365559756756, 0.07069613039493561, -0.028849728405475616, -0.04724215343594551, -0.0934894010424614, -0.1059950739145279, 0.0669117122888565, 0.012815272435545921, 0.022690586745738983], "cos_Ac_negg": [0.10487774759531021, 0.017078710719943047, -0.013036027550697327, -0.0992920845746994, -0.003689222037792206, 0.034252963960170746, -0.04259507730603218, -0.02045252174139023, -0.045132823288440704, 0.025059916079044342, 0.08817455917596817, 0.010035283863544464, 0.06160152703523636, 0.014218760654330254, -0.00823059119284153, -0.06906627863645554, 0.004505272023379803, 0.026955002918839455, -0.03240483999252319, 0.022147294133901596, -0.018964610993862152, 0.050118640065193176, -0.04241577535867691, -0.03627433627843857, -0.01068196166306734, -0.0005757552571594715, -0.07351330667734146, 0.0071894340217113495, -0.0036361757665872574, -0.04875703901052475, 0.06124519556760788, 0.041685089468955994, 0.02372598648071289, -0.06642483919858932, -0.07185755670070648, -0.034029535949230194, -0.04177357256412506, -0.03938595950603485, 0.02855924889445305, -0.011848047375679016, -0.016205070540308952, 0.043828852474689484, 0.024598635733127594, -0.024466268718242645, -0.044855717569589615, 0.057504862546920776, 0.04564825817942619, -0.010068122297525406, 0.08641175180673599, 0.12965255975723267, 0.05013319104909897, 0.01800365559756756, 0.07069613039493561, -0.028849728405475616, -0.04724215343594551, -0.0934894010424614, -0.1059950739145279, 0.0669117122888565, 0.012815272435545921, 0.022690586745738983], "r_norm": [2.513712167739868, 2.2925095558166504, 2.503589153289795, 2.4405970573425293, 2.4142374992370605, 2.307804584503174, 2.491546630859375, 2.3203177452087402, 2.420292377471924, 2.3543174266815186, 2.275521755218506, 2.3005950450897217, 2.4583921432495117, 2.4278013706207275, 2.490736961364746, 2.4330320358276367, 2.5697293281555176, 2.2787208557128906, 2.414294719696045, 2.3761770725250244, 2.4508726596832275, 2.451340675354004, 2.3305158615112305, 2.3771731853485107, 2.419008493423462, 2.4878344535827637, 2.5716605186462402, 2.4627885818481445, 2.343726634979248, 2.426025867462158, 2.400932550430298, 2.334235191345215, 2.3967347145080566, 2.5428006649017334, 2.4857826232910156, 2.344851493835449, 2.314310073852539, 2.32454514503479, 2.4075112342834473, 2.3712234497070312, 2.46826171875, 2.4232234954833984, 2.337924003601074, 2.4635753631591797, 2.436523675918579, 2.4753594398498535, 2.3852529525756836, 2.33897066116333, 2.548658847808838, 2.427408218383789, 2.3551411628723145, 2.3422157764434814, 2.4167022705078125, 2.4697623252868652, 2.4194817543029785, 2.3368473052978516, 2.2841756343841553, 2.4974498748779297, 2.3617746829986572, 2.376339912414551]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3363, "test_loss": 1.8859697387695313}, {"step": 500, "epoch": 1, "train_loss": 1.8172451257705688, "cos_r_negg": [0.5054040551185608, 0.04717698693275452, -0.013306602835655212, -0.016469061374664307, -0.04374157637357712, 0.03563360869884491, 0.021355710923671722, 0.03413999080657959, -0.02129669487476349, 0.05615711957216263, 0.019830049946904182, -0.06207661330699921, -0.018909761682152748, -0.034363437443971634, 0.07669782638549805, 0.03694828599691391, -0.040620505809783936, 0.024687791243195534, 0.018053723499178886, 0.09880384802818298, 0.02923905849456787, 0.004882264882326126, 0.021349182352423668, -0.03176683932542801, -0.03185686096549034, -0.018600594252347946, -0.023207899183034897, -0.0031171038281172514, 0.03187406063079834, 0.054896946996450424, 0.03992067649960518, 0.049270544201135635, -0.025424331426620483, -0.01134447567164898, 0.05241583287715912, -0.06947173178195953, -0.0075959209352731705, 0.026430848985910416, -0.007962550967931747, -0.012401909567415714, -0.03006584197282791, 0.014377336017787457, -0.006534600164741278, -0.02004932053387165, -0.015075746923685074, 0.02172667160630226, 0.05386526882648468, -0.09798265993595123, 0.016570886597037315, -0.05556435137987137, -0.0017419992946088314, -0.002373497234657407, 0.02299337089061737, 0.008164988830685616, -0.009039366617798805, -0.025090156123042107, -0.006325378082692623, 0.10060054808855057, -0.08078669756650925, -0.010532718151807785], "cos_innovation_negg": [0.5054040551185608, 0.04717698693275452, -0.013306602835655212, -0.016469061374664307, -0.04374157637357712, 0.03563360869884491, 0.021355710923671722, 0.03413999080657959, -0.02129669487476349, 0.05615711957216263, 0.019830049946904182, -0.06207661330699921, -0.018909761682152748, -0.034363437443971634, 0.07669782638549805, 0.03694828599691391, -0.040620505809783936, 0.024687791243195534, 0.018053723499178886, 0.09880384802818298, 0.02923905849456787, 0.004882264882326126, 0.021349182352423668, -0.03176683932542801, -0.03185686096549034, -0.018600594252347946, -0.023207899183034897, -0.0031171038281172514, 0.03187406063079834, 0.054896946996450424, 0.03992067649960518, 0.049270544201135635, -0.025424331426620483, -0.01134447567164898, 0.05241583287715912, -0.06947173178195953, -0.0075959209352731705, 0.026430848985910416, -0.007962550967931747, -0.012401909567415714, -0.03006584197282791, 0.014377336017787457, -0.006534600164741278, -0.02004932053387165, -0.015075746923685074, 0.02172667160630226, 0.05386526882648468, -0.09798265993595123, 0.016570886597037315, -0.05556435137987137, -0.0017419992946088314, -0.002373497234657407, 0.02299337089061737, 0.008164988830685616, -0.009039366617798805, -0.025090156123042107, -0.006325378082692623, 0.10060054808855057, -0.08078669756650925, -0.010532718151807785], "cos_apical_negg": [0.5054040551185608, 0.04717698693275452, -0.013306602835655212, -0.016469061374664307, -0.04374157637357712, 0.03563360869884491, 0.021355710923671722, 0.03413999080657959, -0.02129669487476349, 0.05615711957216263, 0.019830049946904182, -0.06207661330699921, -0.018909761682152748, -0.034363437443971634, 0.07669782638549805, 0.03694828599691391, -0.040620505809783936, 0.024687791243195534, 0.018053723499178886, 0.09880384802818298, 0.02923905849456787, 0.004882264882326126, 0.021349182352423668, -0.03176683932542801, -0.03185686096549034, -0.018600594252347946, -0.023207899183034897, -0.0031171038281172514, 0.03187406063079834, 0.054896946996450424, 0.03992067649960518, 0.049270544201135635, -0.025424331426620483, -0.01134447567164898, 0.05241583287715912, -0.06947173178195953, -0.0075959209352731705, 0.026430848985910416, -0.007962550967931747, -0.012401909567415714, -0.03006584197282791, 0.014377336017787457, -0.006534600164741278, -0.02004932053387165, -0.015075746923685074, 0.02172667160630226, 0.05386526882648468, -0.09798265993595123, 0.016570886597037315, -0.05556435137987137, -0.0017419992946088314, -0.002373497234657407, 0.02299337089061737, 0.008164988830685616, -0.009039366617798805, -0.025090156123042107, -0.006325378082692623, 0.10060054808855057, -0.08078669756650925, -0.010532718151807785], "cos_Ac_negg": [0.5054040551185608, 0.04717698693275452, -0.013306602835655212, -0.016469061374664307, -0.04374157637357712, 0.03563360869884491, 0.021355710923671722, 0.03413999080657959, -0.02129669487476349, 0.05615711957216263, 0.019830049946904182, -0.06207661330699921, -0.018909761682152748, -0.034363437443971634, 0.07669782638549805, 0.03694828599691391, -0.040620505809783936, 0.024687791243195534, 0.018053723499178886, 0.09880384802818298, 0.02923905849456787, 0.004882264882326126, 0.021349182352423668, -0.03176683932542801, -0.03185686096549034, -0.018600594252347946, -0.023207899183034897, -0.0031171038281172514, 0.03187406063079834, 0.054896946996450424, 0.03992067649960518, 0.049270544201135635, -0.025424331426620483, -0.01134447567164898, 0.05241583287715912, -0.06947173178195953, -0.0075959209352731705, 0.026430848985910416, -0.007962550967931747, -0.012401909567415714, -0.03006584197282791, 0.014377336017787457, -0.006534600164741278, -0.02004932053387165, -0.015075746923685074, 0.02172667160630226, 0.05386526882648468, -0.09798265993595123, 0.016570886597037315, -0.05556435137987137, -0.0017419992946088314, -0.002373497234657407, 0.02299337089061737, 0.008164988830685616, -0.009039366617798805, -0.025090156123042107, -0.006325378082692623, 0.10060054808855057, -0.08078669756650925, -0.010532718151807785], "r_norm": [2.272214889526367, 2.081476926803589, 2.2829720973968506, 2.22816801071167, 2.2080883979797363, 2.092803716659546, 2.2387866973876953, 2.0982394218444824, 2.186400890350342, 2.149167060852051, 2.0620245933532715, 2.0712854862213135, 2.236693859100342, 2.2169532775878906, 2.273493528366089, 2.2309985160827637, 2.3391964435577393, 2.062960386276245, 2.1912295818328857, 2.165156364440918, 2.202676773071289, 2.255037784576416, 2.1311984062194824, 2.152338743209839, 2.208352565765381, 2.2830252647399902, 2.3248403072357178, 2.2299747467041016, 2.1187686920166016, 2.2226202487945557, 2.157104253768921, 2.116624593734741, 2.192584753036499, 2.3390464782714844, 2.2790560722351074, 2.1324031352996826, 2.0667190551757812, 2.1110501289367676, 2.2057132720947266, 2.163672924041748, 2.250622272491455, 2.221928119659424, 2.129903793334961, 2.194671392440796, 2.200866460800171, 2.2851741313934326, 2.163421630859375, 2.1326074600219727, 2.3107714653015137, 2.182701587677002, 2.1454992294311523, 2.1274142265319824, 2.203202724456787, 2.2616522312164307, 2.2191162109375, 2.164818525314331, 2.086693048477173, 2.2617743015289307, 2.1601762771606445, 2.1698060035705566]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3603, "test_loss": 1.8569560791015625}, {"step": 1000, "epoch": 2, "train_loss": 1.8097293376922607, "cos_r_negg": [0.5982925891876221, 0.03378725424408913, -0.010709518566727638, -0.02001176029443741, 0.00020664697512984276, 0.017155956476926804, -0.0024581262841820717, 0.06244847550988197, 0.01826554723083973, -0.014077657833695412, 0.01884012296795845, -0.043855614960193634, -0.026568297296762466, -0.006792666390538216, 0.03052903339266777, 0.04919211566448212, 0.05196297541260719, -0.007215067278593779, 0.06526380032300949, 0.06697715073823929, 0.0003561279736459255, 0.03148699924349785, 0.012303957715630531, -0.054757218807935715, 0.04360736906528473, 0.014125930145382881, -0.012898901477456093, 0.0346953459084034, 0.015383305959403515, 0.017260173335671425, 0.04752572253346443, 0.019092507660388947, 0.0002500605769455433, -0.04350917041301727, 0.04176851734519005, -0.03449280560016632, 0.005341366399079561, 0.022653575986623764, 0.05682589113712311, -0.009958753362298012, 0.04950485751032829, 0.02204926311969757, -0.03959440439939499, 0.0010628420859575272, 0.008776875212788582, 0.036754004657268524, 0.005581468343734741, -0.10428312420845032, -0.02329966425895691, -0.05940809100866318, 0.014329937286674976, -0.03114517591893673, 0.04996931925415993, 0.02880464866757393, -0.02593836933374405, 0.0020137964747846127, 0.03406897932291031, 0.06187107786536217, -0.04722806066274643, 0.047361940145492554], "cos_innovation_negg": [0.5982925891876221, 0.03378725424408913, -0.010709518566727638, -0.02001176029443741, 0.00020664697512984276, 0.017155956476926804, -0.0024581262841820717, 0.06244847550988197, 0.01826554723083973, -0.014077657833695412, 0.01884012296795845, -0.043855614960193634, -0.026568297296762466, -0.006792666390538216, 0.03052903339266777, 0.04919211566448212, 0.05196297541260719, -0.007215067278593779, 0.06526380032300949, 0.06697715073823929, 0.0003561279736459255, 0.03148699924349785, 0.012303957715630531, -0.054757218807935715, 0.04360736906528473, 0.014125930145382881, -0.012898901477456093, 0.0346953459084034, 0.015383305959403515, 0.017260173335671425, 0.04752572253346443, 0.019092507660388947, 0.0002500605769455433, -0.04350917041301727, 0.04176851734519005, -0.03449280560016632, 0.005341366399079561, 0.022653575986623764, 0.05682589113712311, -0.009958753362298012, 0.04950485751032829, 0.02204926311969757, -0.03959440439939499, 0.0010628420859575272, 0.008776875212788582, 0.036754004657268524, 0.005581468343734741, -0.10428312420845032, -0.02329966425895691, -0.05940809100866318, 0.014329937286674976, -0.03114517591893673, 0.04996931925415993, 0.02880464866757393, -0.02593836933374405, 0.0020137964747846127, 0.03406897932291031, 0.06187107786536217, -0.04722806066274643, 0.047361940145492554], "cos_apical_negg": [0.5982925891876221, 0.03378725424408913, -0.010709518566727638, -0.02001176029443741, 0.00020664697512984276, 0.017155956476926804, -0.0024581262841820717, 0.06244847550988197, 0.01826554723083973, -0.014077657833695412, 0.01884012296795845, -0.043855614960193634, -0.026568297296762466, -0.006792666390538216, 0.03052903339266777, 0.04919211566448212, 0.05196297541260719, -0.007215067278593779, 0.06526380032300949, 0.06697715073823929, 0.0003561279736459255, 0.03148699924349785, 0.012303957715630531, -0.054757218807935715, 0.04360736906528473, 0.014125930145382881, -0.012898901477456093, 0.0346953459084034, 0.015383305959403515, 0.017260173335671425, 0.04752572253346443, 0.019092507660388947, 0.0002500605769455433, -0.04350917041301727, 0.04176851734519005, -0.03449280560016632, 0.005341366399079561, 0.022653575986623764, 0.05682589113712311, -0.009958753362298012, 0.04950485751032829, 0.02204926311969757, -0.03959440439939499, 0.0010628420859575272, 0.008776875212788582, 0.036754004657268524, 0.005581468343734741, -0.10428312420845032, -0.02329966425895691, -0.05940809100866318, 0.014329937286674976, -0.03114517591893673, 0.04996931925415993, 0.02880464866757393, -0.02593836933374405, 0.0020137964747846127, 0.03406897932291031, 0.06187107786536217, -0.04722806066274643, 0.047361940145492554], "cos_Ac_negg": [0.5982925891876221, 0.03378725424408913, -0.010709518566727638, -0.02001176029443741, 0.00020664697512984276, 0.017155956476926804, -0.0024581262841820717, 0.06244847550988197, 0.01826554723083973, -0.014077657833695412, 0.01884012296795845, -0.043855614960193634, -0.026568297296762466, -0.006792666390538216, 0.03052903339266777, 0.04919211566448212, 0.05196297541260719, -0.007215067278593779, 0.06526380032300949, 0.06697715073823929, 0.0003561279736459255, 0.03148699924349785, 0.012303957715630531, -0.054757218807935715, 0.04360736906528473, 0.014125930145382881, -0.012898901477456093, 0.0346953459084034, 0.015383305959403515, 0.017260173335671425, 0.04752572253346443, 0.019092507660388947, 0.0002500605769455433, -0.04350917041301727, 0.04176851734519005, -0.03449280560016632, 0.005341366399079561, 0.022653575986623764, 0.05682589113712311, -0.009958753362298012, 0.04950485751032829, 0.02204926311969757, -0.03959440439939499, 0.0010628420859575272, 0.008776875212788582, 0.036754004657268524, 0.005581468343734741, -0.10428312420845032, -0.02329966425895691, -0.05940809100866318, 0.014329937286674976, -0.03114517591893673, 0.04996931925415993, 0.02880464866757393, -0.02593836933374405, 0.0020137964747846127, 0.03406897932291031, 0.06187107786536217, -0.04722806066274643, 0.047361940145492554], "r_norm": [2.236823320388794, 2.0596296787261963, 2.2395215034484863, 2.1925013065338135, 2.1743130683898926, 2.0629935264587402, 2.2139244079589844, 2.0700573921203613, 2.1546528339385986, 2.117635488510132, 2.0299973487854004, 2.046137571334839, 2.200767755508423, 2.1830339431762695, 2.2340967655181885, 2.19384765625, 2.2989842891693115, 2.034719944000244, 2.1573328971862793, 2.123563528060913, 2.1663594245910645, 2.215792179107666, 2.0955872535705566, 2.128268241882324, 2.1762726306915283, 2.2452080249786377, 2.2989513874053955, 2.196718454360962, 2.080994129180908, 2.1865882873535156, 2.1392202377319336, 2.0787882804870605, 2.157473087310791, 2.3042213916778564, 2.2498748302459717, 2.095461845397949, 2.0359582901000977, 2.080470561981201, 2.1629557609558105, 2.1273691654205322, 2.2126660346984863, 2.1915955543518066, 2.1045663356781006, 2.170258045196533, 2.1659836769104004, 2.248746871948242, 2.134227752685547, 2.1015141010284424, 2.2804646492004395, 2.1504220962524414, 2.117459297180176, 2.0951874256134033, 2.175931930541992, 2.2290093898773193, 2.1788434982299805, 2.133422613143921, 2.0594043731689453, 2.2278923988342285, 2.1347756385803223, 2.138667583465576]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.339, "test_loss": 1.8636215576171875}, {"step": 1500, "epoch": 3, "train_loss": 1.8133920431137085, "cos_r_negg": [0.5778146982192993, 0.04067210853099823, 0.014603408984839916, -0.023322943598031998, -0.03171102702617645, 0.031126510351896286, 0.024676062166690826, 0.011806625872850418, 0.004089484456926584, 0.010962286032736301, 0.023552360013127327, -0.016333183273673058, -0.00022935541346669197, 0.0012384681031107903, 0.011708245612680912, 0.05471471697092056, 0.05377234145998955, 0.01660061627626419, 0.033316537737846375, 0.019077198579907417, 0.035762809216976166, 0.033861663192510605, 0.045404523611068726, -0.06916201114654541, 0.07877080142498016, -0.019529635086655617, -0.014672920107841492, 0.03475739061832428, 0.065090112388134, 0.015547014772891998, 0.06935645639896393, 0.016681715846061707, -0.03850594535470009, 0.027319014072418213, 0.02766617387533188, -0.05719796568155289, 0.006088303402066231, -0.002789916703477502, 0.08474171161651611, 0.00699181342497468, 0.062488168478012085, 0.08026745915412903, -0.01051196176558733, 0.005814698524773121, -0.0351269394159317, 0.061275698244571686, 0.030993644148111343, -0.025427943095564842, -0.05548151955008507, -0.050266705453395844, 0.04473815858364105, -0.03675664961338043, 0.04325823485851288, 0.07996921241283417, -0.042686063796281815, -0.017169615253806114, 0.07041385024785995, 0.06498074531555176, -0.04199787229299545, 0.023701556026935577], "cos_innovation_negg": [0.5778146982192993, 0.04067210853099823, 0.014603408984839916, -0.023322943598031998, -0.03171102702617645, 0.031126510351896286, 0.024676062166690826, 0.011806625872850418, 0.004089484456926584, 0.010962286032736301, 0.023552360013127327, -0.016333183273673058, -0.00022935541346669197, 0.0012384681031107903, 0.011708245612680912, 0.05471471697092056, 0.05377234145998955, 0.01660061627626419, 0.033316537737846375, 0.019077198579907417, 0.035762809216976166, 0.033861663192510605, 0.045404523611068726, -0.06916201114654541, 0.07877080142498016, -0.019529635086655617, -0.014672920107841492, 0.03475739061832428, 0.065090112388134, 0.015547014772891998, 0.06935645639896393, 0.016681715846061707, -0.03850594535470009, 0.027319014072418213, 0.02766617387533188, -0.05719796568155289, 0.006088303402066231, -0.002789916703477502, 0.08474171161651611, 0.00699181342497468, 0.062488168478012085, 0.08026745915412903, -0.01051196176558733, 0.005814698524773121, -0.0351269394159317, 0.061275698244571686, 0.030993644148111343, -0.025427943095564842, -0.05548151955008507, -0.050266705453395844, 0.04473815858364105, -0.03675664961338043, 0.04325823485851288, 0.07996921241283417, -0.042686063796281815, -0.017169615253806114, 0.07041385024785995, 0.06498074531555176, -0.04199787229299545, 0.023701556026935577], "cos_apical_negg": [0.5778146982192993, 0.04067210853099823, 0.014603408984839916, -0.023322943598031998, -0.03171102702617645, 0.031126510351896286, 0.024676062166690826, 0.011806625872850418, 0.004089484456926584, 0.010962286032736301, 0.023552360013127327, -0.016333183273673058, -0.00022935541346669197, 0.0012384681031107903, 0.011708245612680912, 0.05471471697092056, 0.05377234145998955, 0.01660061627626419, 0.033316537737846375, 0.019077198579907417, 0.035762809216976166, 0.033861663192510605, 0.045404523611068726, -0.06916201114654541, 0.07877080142498016, -0.019529635086655617, -0.014672920107841492, 0.03475739061832428, 0.065090112388134, 0.015547014772891998, 0.06935645639896393, 0.016681715846061707, -0.03850594535470009, 0.027319014072418213, 0.02766617387533188, -0.05719796568155289, 0.006088303402066231, -0.002789916703477502, 0.08474171161651611, 0.00699181342497468, 0.062488168478012085, 0.08026745915412903, -0.01051196176558733, 0.005814698524773121, -0.0351269394159317, 0.061275698244571686, 0.030993644148111343, -0.025427943095564842, -0.05548151955008507, -0.050266705453395844, 0.04473815858364105, -0.03675664961338043, 0.04325823485851288, 0.07996921241283417, -0.042686063796281815, -0.017169615253806114, 0.07041385024785995, 0.06498074531555176, -0.04199787229299545, 0.023701556026935577], "cos_Ac_negg": [0.5778146982192993, 0.04067210853099823, 0.014603408984839916, -0.023322943598031998, -0.03171102702617645, 0.031126510351896286, 0.024676062166690826, 0.011806625872850418, 0.004089484456926584, 0.010962286032736301, 0.023552360013127327, -0.016333183273673058, -0.00022935541346669197, 0.0012384681031107903, 0.011708245612680912, 0.05471471697092056, 0.05377234145998955, 0.01660061627626419, 0.033316537737846375, 0.019077198579907417, 0.035762809216976166, 0.033861663192510605, 0.045404523611068726, -0.06916201114654541, 0.07877080142498016, -0.019529635086655617, -0.014672920107841492, 0.03475739061832428, 0.065090112388134, 0.015547014772891998, 0.06935645639896393, 0.016681715846061707, -0.03850594535470009, 0.027319014072418213, 0.02766617387533188, -0.05719796568155289, 0.006088303402066231, -0.002789916703477502, 0.08474171161651611, 0.00699181342497468, 0.062488168478012085, 0.08026745915412903, -0.01051196176558733, 0.005814698524773121, -0.0351269394159317, 0.061275698244571686, 0.030993644148111343, -0.025427943095564842, -0.05548151955008507, -0.050266705453395844, 0.04473815858364105, -0.03675664961338043, 0.04325823485851288, 0.07996921241283417, -0.042686063796281815, -0.017169615253806114, 0.07041385024785995, 0.06498074531555176, -0.04199787229299545, 0.023701556026935577], "r_norm": [2.267338752746582, 2.088749647140503, 2.270571231842041, 2.2282934188842773, 2.204601287841797, 2.0848817825317383, 2.243159055709839, 2.0929784774780273, 2.187797784805298, 2.1381397247314453, 2.056943893432617, 2.0771472454071045, 2.2399609088897705, 2.209923028945923, 2.2626242637634277, 2.2286202907562256, 2.3317251205444336, 2.0608158111572266, 2.1954214572906494, 2.1462748050689697, 2.1931135654449463, 2.250808000564575, 2.1242337226867676, 2.15856671333313, 2.2046265602111816, 2.2768545150756836, 2.329680919647217, 2.2236480712890625, 2.116830348968506, 2.218414783477783, 2.1724729537963867, 2.1083648204803467, 2.1898341178894043, 2.323444366455078, 2.277981758117676, 2.1325430870056152, 2.0657334327697754, 2.1075522899627686, 2.188973903656006, 2.1586990356445312, 2.2366132736206055, 2.2175018787384033, 2.1381211280822754, 2.2152395248413086, 2.194903612136841, 2.271186351776123, 2.160344362258911, 2.126209259033203, 2.303607940673828, 2.1791486740112305, 2.1468400955200195, 2.125999927520752, 2.195707082748413, 2.2633509635925293, 2.210883617401123, 2.152672529220581, 2.0949459075927734, 2.257892608642578, 2.1595401763916016, 2.166194438934326]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3468, "test_loss": 1.8772449340820312}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3671, "test_loss": 1.8296071533203124}], "final": {"test_acc": 0.3671, "test_loss": 1.8296071533203124, "wall_s": 57.483853578567505, "cos_r_negg": [0.6026690006256104, 0.03569790720939636, 0.005465397611260414, -0.02040005475282669, 0.0035228431224823, 0.030612843111157417, 0.033377811312675476, 0.018665198236703873, 0.025836581364274025, 0.026464762166142464, 0.02472522109746933, -0.040654826909303665, -0.023752234876155853, -0.017534170299768448, 0.05726249888539314, 0.03450862318277359, 0.036143992096185684, 0.046841710805892944, 0.07423040270805359, 0.01810389570891857, 0.015721088275313377, 0.07469154894351959, 0.03775313124060631, -0.0537196546792984, 0.01733379438519478, 0.01689155399799347, -0.011807186529040337, 0.031777530908584595, 0.05541872978210449, 0.07634670287370682, 0.0041962494142353535, 0.04888368770480156, -0.034009598195552826, 0.01990911364555359, -0.002217576839029789, -0.01397725846618414, 0.03849094733595848, 0.032169777899980545, 0.05368480086326599, 0.013577194884419441, 0.004206962883472443, 0.010362978093326092, 0.004679153207689524, -0.0112288948148489, -0.045124176889657974, 0.09878894686698914, -0.006263609044253826, -0.0477064773440361, -0.020333506166934967, -0.032808560878038406, -0.03397134318947792, -0.02804010920226574, 0.09191104769706726, 0.08185486495494843, -0.030850280076265335, -0.039040300995111465, 0.0859619528055191, 0.05088172107934952, -0.04311472177505493, 0.04487689584493637], "cos_innovation_negg": [0.6026690006256104, 0.03569790720939636, 0.005465397611260414, -0.02040005475282669, 0.0035228431224823, 0.030612843111157417, 0.033377811312675476, 0.018665198236703873, 0.025836581364274025, 0.026464762166142464, 0.02472522109746933, -0.040654826909303665, -0.023752234876155853, -0.017534170299768448, 0.05726249888539314, 0.03450862318277359, 0.036143992096185684, 0.046841710805892944, 0.07423040270805359, 0.01810389570891857, 0.015721088275313377, 0.07469154894351959, 0.03775313124060631, -0.0537196546792984, 0.01733379438519478, 0.01689155399799347, -0.011807186529040337, 0.031777530908584595, 0.05541872978210449, 0.07634670287370682, 0.0041962494142353535, 0.04888368770480156, -0.034009598195552826, 0.01990911364555359, -0.002217576839029789, -0.01397725846618414, 0.03849094733595848, 0.032169777899980545, 0.05368480086326599, 0.013577194884419441, 0.004206962883472443, 0.010362978093326092, 0.004679153207689524, -0.0112288948148489, -0.045124176889657974, 0.09878894686698914, -0.006263609044253826, -0.0477064773440361, -0.020333506166934967, -0.032808560878038406, -0.03397134318947792, -0.02804010920226574, 0.09191104769706726, 0.08185486495494843, -0.030850280076265335, -0.039040300995111465, 0.0859619528055191, 0.05088172107934952, -0.04311472177505493, 0.04487689584493637], "cos_apical_negg": [0.6026690006256104, 0.03569790720939636, 0.005465397611260414, -0.02040005475282669, 0.0035228431224823, 0.030612843111157417, 0.033377811312675476, 0.018665198236703873, 0.025836581364274025, 0.026464762166142464, 0.02472522109746933, -0.040654826909303665, -0.023752234876155853, -0.017534170299768448, 0.05726249888539314, 0.03450862318277359, 0.036143992096185684, 0.046841710805892944, 0.07423040270805359, 0.01810389570891857, 0.015721088275313377, 0.07469154894351959, 0.03775313124060631, -0.0537196546792984, 0.01733379438519478, 0.01689155399799347, -0.011807186529040337, 0.031777530908584595, 0.05541872978210449, 0.07634670287370682, 0.0041962494142353535, 0.04888368770480156, -0.034009598195552826, 0.01990911364555359, -0.002217576839029789, -0.01397725846618414, 0.03849094733595848, 0.032169777899980545, 0.05368480086326599, 0.013577194884419441, 0.004206962883472443, 0.010362978093326092, 0.004679153207689524, -0.0112288948148489, -0.045124176889657974, 0.09878894686698914, -0.006263609044253826, -0.0477064773440361, -0.020333506166934967, -0.032808560878038406, -0.03397134318947792, -0.02804010920226574, 0.09191104769706726, 0.08185486495494843, -0.030850280076265335, -0.039040300995111465, 0.0859619528055191, 0.05088172107934952, -0.04311472177505493, 0.04487689584493637], "cos_Ac_negg": [0.6026690006256104, 0.03569790720939636, 0.005465397611260414, -0.02040005475282669, 0.0035228431224823, 0.030612843111157417, 0.033377811312675476, 0.018665198236703873, 0.025836581364274025, 0.026464762166142464, 0.02472522109746933, -0.040654826909303665, -0.023752234876155853, -0.017534170299768448, 0.05726249888539314, 0.03450862318277359, 0.036143992096185684, 0.046841710805892944, 0.07423040270805359, 0.01810389570891857, 0.015721088275313377, 0.07469154894351959, 0.03775313124060631, -0.0537196546792984, 0.01733379438519478, 0.01689155399799347, -0.011807186529040337, 0.031777530908584595, 0.05541872978210449, 0.07634670287370682, 0.0041962494142353535, 0.04888368770480156, -0.034009598195552826, 0.01990911364555359, -0.002217576839029789, -0.01397725846618414, 0.03849094733595848, 0.032169777899980545, 0.05368480086326599, 0.013577194884419441, 0.004206962883472443, 0.010362978093326092, 0.004679153207689524, -0.0112288948148489, -0.045124176889657974, 0.09878894686698914, -0.006263609044253826, -0.0477064773440361, -0.020333506166934967, -0.032808560878038406, -0.03397134318947792, -0.02804010920226574, 0.09191104769706726, 0.08185486495494843, -0.030850280076265335, -0.039040300995111465, 0.0859619528055191, 0.05088172107934952, -0.04311472177505493, 0.04487689584493637]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d60_s3.json b/results/scale_v3_cifar10_dfa_w64_d60_s3.json new file mode 100644 index 0000000..da7edb4 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d60_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d60_s3", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.416471481323242, "cos_r_negg": [0.011034354567527771, 0.009778410196304321, -0.005175890401005745, 0.06470051407814026, 0.018282506614923477, 0.011382738128304482, -0.005024812649935484, -0.019846880808472633, -0.009526850655674934, 0.033607300370931625, 0.056432295590639114, 0.007911688648164272, 0.031163012608885765, -0.06069733202457428, 0.025152752175927162, -0.07322832942008972, -0.035255808383226395, 0.07395307719707489, 0.028051743283867836, 0.006810421124100685, 0.02829650789499283, -0.018351973965764046, -0.05066262185573578, 0.03387511149048805, 0.043458569794893265, 0.049599677324295044, 0.05521861091256142, -0.03410328924655914, 0.005314301699399948, -0.024868059903383255, -0.032576486468315125, 0.042438067495822906, 0.07566317170858383, 0.035273849964141846, -0.007690399419516325, 0.002642424777150154, -0.11291581392288208, 0.052527494728565216, 0.023594997823238373, -0.010310381650924683, 0.037106551229953766, 0.016276765614748, 0.05549519509077072, -0.026969514787197113, -0.007964426651597023, 0.07981345802545547, 0.022261226549744606, 0.0497598722577095, -0.05376143753528595, 0.026926986873149872, -0.0004265345633029938, -0.055940743535757065, 0.026235613971948624, 0.051641568541526794, -0.045333363115787506, 0.050232961773872375, 0.029803914949297905, 0.02461191825568676, -0.013203268870711327, 0.03761856630444527], "cos_innovation_negg": [0.011034354567527771, 0.009778410196304321, -0.005175890401005745, 0.06470051407814026, 0.018282506614923477, 0.011382738128304482, -0.005024812649935484, -0.019846880808472633, -0.009526850655674934, 0.033607300370931625, 0.056432295590639114, 0.007911688648164272, 0.031163012608885765, -0.06069733202457428, 0.025152752175927162, -0.07322832942008972, -0.035255808383226395, 0.07395307719707489, 0.028051743283867836, 0.006810421124100685, 0.02829650789499283, -0.018351973965764046, -0.05066262185573578, 0.03387511149048805, 0.043458569794893265, 0.049599677324295044, 0.05521861091256142, -0.03410328924655914, 0.005314301699399948, -0.024868059903383255, -0.032576486468315125, 0.042438067495822906, 0.07566317170858383, 0.035273849964141846, -0.007690399419516325, 0.002642424777150154, -0.11291581392288208, 0.052527494728565216, 0.023594997823238373, -0.010310381650924683, 0.037106551229953766, 0.016276765614748, 0.05549519509077072, -0.026969514787197113, -0.007964426651597023, 0.07981345802545547, 0.022261226549744606, 0.0497598722577095, -0.05376143753528595, 0.026926986873149872, -0.0004265345633029938, -0.055940743535757065, 0.026235613971948624, 0.051641568541526794, -0.045333363115787506, 0.050232961773872375, 0.029803914949297905, 0.02461191825568676, -0.013203268870711327, 0.03761856630444527], "cos_apical_negg": [0.011034354567527771, 0.009778410196304321, -0.005175890401005745, 0.06470051407814026, 0.018282506614923477, 0.011382738128304482, -0.005024812649935484, -0.019846880808472633, -0.009526850655674934, 0.033607300370931625, 0.056432295590639114, 0.007911688648164272, 0.031163012608885765, -0.06069733202457428, 0.025152752175927162, -0.07322832942008972, -0.035255808383226395, 0.07395307719707489, 0.028051743283867836, 0.006810421124100685, 0.02829650789499283, -0.018351973965764046, -0.05066262185573578, 0.03387511149048805, 0.043458569794893265, 0.049599677324295044, 0.05521861091256142, -0.03410328924655914, 0.005314301699399948, -0.024868059903383255, -0.032576486468315125, 0.042438067495822906, 0.07566317170858383, 0.035273849964141846, -0.007690399419516325, 0.002642424777150154, -0.11291581392288208, 0.052527494728565216, 0.023594997823238373, -0.010310381650924683, 0.037106551229953766, 0.016276765614748, 0.05549519509077072, -0.026969514787197113, -0.007964426651597023, 0.07981345802545547, 0.022261226549744606, 0.0497598722577095, -0.05376143753528595, 0.026926986873149872, -0.0004265345633029938, -0.055940743535757065, 0.026235613971948624, 0.051641568541526794, -0.045333363115787506, 0.050232961773872375, 0.029803914949297905, 0.02461191825568676, -0.013203268870711327, 0.03761856630444527], "cos_Ac_negg": [0.011034354567527771, 0.009778410196304321, -0.005175890401005745, 0.06470051407814026, 0.018282506614923477, 0.011382738128304482, -0.005024812649935484, -0.019846880808472633, -0.009526850655674934, 0.033607300370931625, 0.056432295590639114, 0.007911688648164272, 0.031163012608885765, -0.06069733202457428, 0.025152752175927162, -0.07322832942008972, -0.035255808383226395, 0.07395307719707489, 0.028051743283867836, 0.006810421124100685, 0.02829650789499283, -0.018351973965764046, -0.05066262185573578, 0.03387511149048805, 0.043458569794893265, 0.049599677324295044, 0.05521861091256142, -0.03410328924655914, 0.005314301699399948, -0.024868059903383255, -0.032576486468315125, 0.042438067495822906, 0.07566317170858383, 0.035273849964141846, -0.007690399419516325, 0.002642424777150154, -0.11291581392288208, 0.052527494728565216, 0.023594997823238373, -0.010310381650924683, 0.037106551229953766, 0.016276765614748, 0.05549519509077072, -0.026969514787197113, -0.007964426651597023, 0.07981345802545547, 0.022261226549744606, 0.0497598722577095, -0.05376143753528595, 0.026926986873149872, -0.0004265345633029938, -0.055940743535757065, 0.026235613971948624, 0.051641568541526794, -0.045333363115787506, 0.050232961773872375, 0.029803914949297905, 0.02461191825568676, -0.013203268870711327, 0.03761856630444527], "r_norm": [2.543682813644409, 2.4681715965270996, 2.382098436355591, 2.3982362747192383, 2.4679696559906006, 2.564542531967163, 2.3697268962860107, 2.5606372356414795, 2.446824550628662, 2.4486100673675537, 2.493159770965576, 2.261246681213379, 2.3780174255371094, 2.545248508453369, 2.46536922454834, 2.4224092960357666, 2.534801483154297, 2.304856061935425, 2.4681167602539062, 2.5735745429992676, 2.604710817337036, 2.5486974716186523, 2.51301908493042, 2.6024398803710938, 2.4404380321502686, 2.42704701423645, 2.3967599868774414, 2.400517702102661, 2.4280176162719727, 2.6205453872680664, 2.5500264167785645, 2.4981589317321777, 2.476635694503784, 2.48075532913208, 2.4816904067993164, 2.4836316108703613, 2.4943137168884277, 2.4834561347961426, 2.502634286880493, 2.347290515899658, 2.480867385864258, 2.478705883026123, 2.4616293907165527, 2.504462718963623, 2.2933263778686523, 2.5474891662597656, 2.3788580894470215, 2.431075096130371, 2.5752921104431152, 2.545206069946289, 2.5327000617980957, 2.419905662536621, 2.3588967323303223, 2.4085493087768555, 2.3420486450195312, 2.38167142868042, 2.4683940410614014, 2.422899007797241, 2.4193553924560547, 2.5130624771118164]}, {"epoch_end": 0, "step": 391, "test_acc": 0.3462, "test_loss": 1.8815731689453126}, {"step": 500, "epoch": 1, "train_loss": 1.7625470161437988, "cos_r_negg": [0.5473410487174988, 0.029749108478426933, 0.040507614612579346, 0.05899553373456001, -0.0009539490565657616, -0.049099747091531754, 0.02956867590546608, 0.034946925938129425, -0.02725764736533165, 0.042042240500450134, -0.04587266966700554, -0.0035246815532445908, 0.025652922689914703, 0.01817459799349308, 0.017357688397169113, 0.0011543402215465903, 0.04390783980488777, -0.02894407883286476, 0.04844854399561882, 0.039135683327913284, -0.04929898679256439, -0.06978000700473785, 0.06919372081756592, -0.029181834310293198, 0.02198691852390766, -0.06511790305376053, 0.018031682819128036, -0.05167689919471741, 0.04889536648988724, -0.0344417467713356, -0.03321873024106026, 0.04132002592086792, 0.0034072990529239178, 0.0641528069972992, 0.05335326865315437, 0.019268689677119255, -0.09292290359735489, 0.021184630692005157, -0.014759203419089317, 0.04832688346505165, -0.05906318128108978, -0.0645274892449379, 0.02925347536802292, -0.0738200843334198, 0.03796761482954025, 0.04775761812925339, -0.11676812171936035, 0.06793846189975739, -0.003982875496149063, -0.030202597379684448, -0.017439551651477814, 0.038238462060689926, -0.08563432097434998, 0.010651003569364548, -0.041275754570961, -0.07457277178764343, 0.003246141131967306, -0.03875042125582695, -0.011117867194116116, 0.007179941516369581], "cos_innovation_negg": [0.5473410487174988, 0.029749108478426933, 0.040507614612579346, 0.05899553373456001, -0.0009539490565657616, -0.049099747091531754, 0.02956867590546608, 0.034946925938129425, -0.02725764736533165, 0.042042240500450134, -0.04587266966700554, -0.0035246815532445908, 0.025652922689914703, 0.01817459799349308, 0.017357688397169113, 0.0011543402215465903, 0.04390783980488777, -0.02894407883286476, 0.04844854399561882, 0.039135683327913284, -0.04929898679256439, -0.06978000700473785, 0.06919372081756592, -0.029181834310293198, 0.02198691852390766, -0.06511790305376053, 0.018031682819128036, -0.05167689919471741, 0.04889536648988724, -0.0344417467713356, -0.03321873024106026, 0.04132002592086792, 0.0034072990529239178, 0.0641528069972992, 0.05335326865315437, 0.019268689677119255, -0.09292290359735489, 0.021184630692005157, -0.014759203419089317, 0.04832688346505165, -0.05906318128108978, -0.0645274892449379, 0.02925347536802292, -0.0738200843334198, 0.03796761482954025, 0.04775761812925339, -0.11676812171936035, 0.06793846189975739, -0.003982875496149063, -0.030202597379684448, -0.017439551651477814, 0.038238462060689926, -0.08563432097434998, 0.010651003569364548, -0.041275754570961, -0.07457277178764343, 0.003246141131967306, -0.03875042125582695, -0.011117867194116116, 0.007179941516369581], "cos_apical_negg": [0.5473410487174988, 0.029749108478426933, 0.040507614612579346, 0.05899553373456001, -0.0009539490565657616, -0.049099747091531754, 0.02956867590546608, 0.034946925938129425, -0.02725764736533165, 0.042042240500450134, -0.04587266966700554, -0.0035246815532445908, 0.025652922689914703, 0.01817459799349308, 0.017357688397169113, 0.0011543402215465903, 0.04390783980488777, -0.02894407883286476, 0.04844854399561882, 0.039135683327913284, -0.04929898679256439, -0.06978000700473785, 0.06919372081756592, -0.029181834310293198, 0.02198691852390766, -0.06511790305376053, 0.018031682819128036, -0.05167689919471741, 0.04889536648988724, -0.0344417467713356, -0.03321873024106026, 0.04132002592086792, 0.0034072990529239178, 0.0641528069972992, 0.05335326865315437, 0.019268689677119255, -0.09292290359735489, 0.021184630692005157, -0.014759203419089317, 0.04832688346505165, -0.05906318128108978, -0.0645274892449379, 0.02925347536802292, -0.0738200843334198, 0.03796761482954025, 0.04775761812925339, -0.11676812171936035, 0.06793846189975739, -0.003982875496149063, -0.030202597379684448, -0.017439551651477814, 0.038238462060689926, -0.08563432097434998, 0.010651003569364548, -0.041275754570961, -0.07457277178764343, 0.003246141131967306, -0.03875042125582695, -0.011117867194116116, 0.007179941516369581], "cos_Ac_negg": [0.5473410487174988, 0.029749108478426933, 0.040507614612579346, 0.05899553373456001, -0.0009539490565657616, -0.049099747091531754, 0.02956867590546608, 0.034946925938129425, -0.02725764736533165, 0.042042240500450134, -0.04587266966700554, -0.0035246815532445908, 0.025652922689914703, 0.01817459799349308, 0.017357688397169113, 0.0011543402215465903, 0.04390783980488777, -0.02894407883286476, 0.04844854399561882, 0.039135683327913284, -0.04929898679256439, -0.06978000700473785, 0.06919372081756592, -0.029181834310293198, 0.02198691852390766, -0.06511790305376053, 0.018031682819128036, -0.05167689919471741, 0.04889536648988724, -0.0344417467713356, -0.03321873024106026, 0.04132002592086792, 0.0034072990529239178, 0.0641528069972992, 0.05335326865315437, 0.019268689677119255, -0.09292290359735489, 0.021184630692005157, -0.014759203419089317, 0.04832688346505165, -0.05906318128108978, -0.0645274892449379, 0.02925347536802292, -0.0738200843334198, 0.03796761482954025, 0.04775761812925339, -0.11676812171936035, 0.06793846189975739, -0.003982875496149063, -0.030202597379684448, -0.017439551651477814, 0.038238462060689926, -0.08563432097434998, 0.010651003569364548, -0.041275754570961, -0.07457277178764343, 0.003246141131967306, -0.03875042125582695, -0.011117867194116116, 0.007179941516369581], "r_norm": [2.2852718830108643, 2.213348865509033, 2.0883994102478027, 2.166900396347046, 2.178363561630249, 2.2761824131011963, 2.1159420013427734, 2.284665107727051, 2.193911075592041, 2.192317247390747, 2.2052512168884277, 2.0256237983703613, 2.1023035049438477, 2.2655158042907715, 2.1850621700286865, 2.147256374359131, 2.250645160675049, 2.082151412963867, 2.201376438140869, 2.2652106285095215, 2.31304931640625, 2.270045280456543, 2.221255302429199, 2.320812702178955, 2.182539939880371, 2.195979118347168, 2.14009428024292, 2.117737293243408, 2.162111759185791, 2.301762342453003, 2.2803802490234375, 2.225593090057373, 2.2037181854248047, 2.2120957374572754, 2.1814894676208496, 2.2071986198425293, 2.228107452392578, 2.2119569778442383, 2.224391460418701, 2.123133659362793, 2.2046942710876465, 2.2149035930633545, 2.148021697998047, 2.212257146835327, 2.024836778640747, 2.257174015045166, 2.1318342685699463, 2.1477837562561035, 2.280033826828003, 2.259819507598877, 2.2575278282165527, 2.1689352989196777, 2.123037099838257, 2.1552557945251465, 2.096240520477295, 2.1276469230651855, 2.198523998260498, 2.1365365982055664, 2.162288188934326, 2.2248334884643555]}, {"epoch_end": 1, "step": 782, "test_acc": 0.359, "test_loss": 1.8630443359375}, {"step": 1000, "epoch": 2, "train_loss": 1.7913938760757446, "cos_r_negg": [0.5971537828445435, 0.021894119679927826, 0.09817145764827728, 0.061049383133649826, -0.029152052477002144, -0.007392494007945061, -0.007802640087902546, 0.04543574899435043, -0.007431368343532085, 0.029778501018881798, 0.02573494240641594, 0.04941509664058685, 0.03152729198336601, 2.6545021682977676e-06, 0.030992280691862106, 0.007582557387650013, -0.010552633553743362, -0.0510983020067215, -0.007346827071160078, 0.030351487919688225, -0.04595271497964859, -0.020877104252576828, 0.07050575315952301, -0.027703961357474327, 0.0633077472448349, -0.024771522730588913, 0.024814501404762268, -0.010597830638289452, 0.034355588257312775, -0.00284773763269186, -0.027902286499738693, 0.0036842701956629753, -0.01354872528463602, 0.04162687063217163, 0.05248928815126419, 0.0010142766404896975, -0.06225695461034775, 0.025511465966701508, -0.038294583559036255, 0.03588205203413963, -0.018996134400367737, -0.0036708172410726547, 0.07859359681606293, -0.08571013063192368, 0.0015851007774472237, 0.010448841378092766, -0.01768655888736248, 0.06259477138519287, 0.00955975241959095, -0.051921457052230835, 0.008461769670248032, 0.06449637562036514, -0.09169629961252213, 0.013119894079864025, -0.010055940598249435, -0.06629485636949539, 0.02068496122956276, -0.05651898682117462, -0.004221017472445965, -0.03782729431986809], "cos_innovation_negg": [0.5971537828445435, 0.021894119679927826, 0.09817145764827728, 0.061049383133649826, -0.029152052477002144, -0.007392494007945061, -0.007802640087902546, 0.04543574899435043, -0.007431368343532085, 0.029778501018881798, 0.02573494240641594, 0.04941509664058685, 0.03152729198336601, 2.6545021682977676e-06, 0.030992280691862106, 0.007582557387650013, -0.010552633553743362, -0.0510983020067215, -0.007346827071160078, 0.030351487919688225, -0.04595271497964859, -0.020877104252576828, 0.07050575315952301, -0.027703961357474327, 0.0633077472448349, -0.024771522730588913, 0.024814501404762268, -0.010597830638289452, 0.034355588257312775, -0.00284773763269186, -0.027902286499738693, 0.0036842701956629753, -0.01354872528463602, 0.04162687063217163, 0.05248928815126419, 0.0010142766404896975, -0.06225695461034775, 0.025511465966701508, -0.038294583559036255, 0.03588205203413963, -0.018996134400367737, -0.0036708172410726547, 0.07859359681606293, -0.08571013063192368, 0.0015851007774472237, 0.010448841378092766, -0.01768655888736248, 0.06259477138519287, 0.00955975241959095, -0.051921457052230835, 0.008461769670248032, 0.06449637562036514, -0.09169629961252213, 0.013119894079864025, -0.010055940598249435, -0.06629485636949539, 0.02068496122956276, -0.05651898682117462, -0.004221017472445965, -0.03782729431986809], "cos_apical_negg": [0.5971537828445435, 0.021894119679927826, 0.09817145764827728, 0.061049383133649826, -0.029152052477002144, -0.007392494007945061, -0.007802640087902546, 0.04543574899435043, -0.007431368343532085, 0.029778501018881798, 0.02573494240641594, 0.04941509664058685, 0.03152729198336601, 2.6545021682977676e-06, 0.030992280691862106, 0.007582557387650013, -0.010552633553743362, -0.0510983020067215, -0.007346827071160078, 0.030351487919688225, -0.04595271497964859, -0.020877104252576828, 0.07050575315952301, -0.027703961357474327, 0.0633077472448349, -0.024771522730588913, 0.024814501404762268, -0.010597830638289452, 0.034355588257312775, -0.00284773763269186, -0.027902286499738693, 0.0036842701956629753, -0.01354872528463602, 0.04162687063217163, 0.05248928815126419, 0.0010142766404896975, -0.06225695461034775, 0.025511465966701508, -0.038294583559036255, 0.03588205203413963, -0.018996134400367737, -0.0036708172410726547, 0.07859359681606293, -0.08571013063192368, 0.0015851007774472237, 0.010448841378092766, -0.01768655888736248, 0.06259477138519287, 0.00955975241959095, -0.051921457052230835, 0.008461769670248032, 0.06449637562036514, -0.09169629961252213, 0.013119894079864025, -0.010055940598249435, -0.06629485636949539, 0.02068496122956276, -0.05651898682117462, -0.004221017472445965, -0.03782729431986809], "cos_Ac_negg": [0.5971537828445435, 0.021894119679927826, 0.09817145764827728, 0.061049383133649826, -0.029152052477002144, -0.007392494007945061, -0.007802640087902546, 0.04543574899435043, -0.007431368343532085, 0.029778501018881798, 0.02573494240641594, 0.04941509664058685, 0.03152729198336601, 2.6545021682977676e-06, 0.030992280691862106, 0.007582557387650013, -0.010552633553743362, -0.0510983020067215, -0.007346827071160078, 0.030351487919688225, -0.04595271497964859, -0.020877104252576828, 0.07050575315952301, -0.027703961357474327, 0.0633077472448349, -0.024771522730588913, 0.024814501404762268, -0.010597830638289452, 0.034355588257312775, -0.00284773763269186, -0.027902286499738693, 0.0036842701956629753, -0.01354872528463602, 0.04162687063217163, 0.05248928815126419, 0.0010142766404896975, -0.06225695461034775, 0.025511465966701508, -0.038294583559036255, 0.03588205203413963, -0.018996134400367737, -0.0036708172410726547, 0.07859359681606293, -0.08571013063192368, 0.0015851007774472237, 0.010448841378092766, -0.01768655888736248, 0.06259477138519287, 0.00955975241959095, -0.051921457052230835, 0.008461769670248032, 0.06449637562036514, -0.09169629961252213, 0.013119894079864025, -0.010055940598249435, -0.06629485636949539, 0.02068496122956276, -0.05651898682117462, -0.004221017472445965, -0.03782729431986809], "r_norm": [2.270751953125, 2.2013630867004395, 2.0757763385772705, 2.1514651775360107, 2.1614019870758057, 2.2729403972625732, 2.0977487564086914, 2.2730135917663574, 2.1904687881469727, 2.180060863494873, 2.195136070251465, 2.013617992401123, 2.093043327331543, 2.2506041526794434, 2.161492347717285, 2.127927780151367, 2.2388346195220947, 2.07600736618042, 2.1931233406066895, 2.2481136322021484, 2.300996780395508, 2.2647571563720703, 2.202075958251953, 2.3077430725097656, 2.166813850402832, 2.182044506072998, 2.128645658493042, 2.1049985885620117, 2.1448745727539062, 2.288891077041626, 2.2687621116638184, 2.2093358039855957, 2.188870429992676, 2.2021546363830566, 2.1689066886901855, 2.18792986869812, 2.2096920013427734, 2.2012839317321777, 2.2067902088165283, 2.1106173992156982, 2.1900229454040527, 2.208646535873413, 2.1376657485961914, 2.2103867530822754, 2.0021584033966064, 2.2566800117492676, 2.124558448791504, 2.1377627849578857, 2.258460760116577, 2.2419233322143555, 2.251932144165039, 2.1588568687438965, 2.1049463748931885, 2.153712272644043, 2.081277847290039, 2.1136860847473145, 2.1732945442199707, 2.1227238178253174, 2.152517795562744, 2.2077884674072266]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3419, "test_loss": 1.8728737182617188}, {"step": 1500, "epoch": 3, "train_loss": 1.7469151020050049, "cos_r_negg": [0.5330359935760498, 0.06855221092700958, 0.05162184685468674, 0.08716437220573425, -0.008329702541232109, -0.008950409479439259, -0.0006544194184243679, 0.06687228381633759, -0.026471810415387154, 0.034034427255392075, -0.0335475318133831, 0.01642283797264099, -0.007982833310961723, 0.0006792694330215454, 0.03442753106355667, 0.02753916010260582, 0.08817972242832184, -0.015462561510503292, 0.05122409760951996, 0.0003430015640333295, -0.01798778772354126, 0.020352086052298546, 0.09322929382324219, -0.05006154254078865, 0.046400655061006546, -0.043441664427518845, 0.012250003404915333, -0.05086598917841911, 0.000632967334240675, -0.02909906953573227, -0.0438908115029335, 0.04437175393104553, -0.0064100902527570724, 0.016334785148501396, 0.06977900117635727, 0.04352376610040665, -0.008959362283349037, 0.0030630086548626423, -0.03147897496819496, 0.0334748774766922, -0.04837929829955101, -0.004705514758825302, -0.004161679185926914, -0.08050394058227539, -0.02144329994916916, 0.034824639558792114, -0.003403794253244996, 0.10653313994407654, -0.013146600686013699, -0.017849180847406387, -0.011511427350342274, 0.0472860261797905, -0.07780762016773224, 0.025587381795048714, 0.024715103209018707, -0.04360129311680794, 0.023186136037111282, -0.03573177754878998, -0.03228802606463432, -0.01966124027967453], "cos_innovation_negg": [0.5330359935760498, 0.06855221092700958, 0.05162184685468674, 0.08716437220573425, -0.008329702541232109, -0.008950409479439259, -0.0006544194184243679, 0.06687228381633759, -0.026471810415387154, 0.034034427255392075, -0.0335475318133831, 0.01642283797264099, -0.007982833310961723, 0.0006792694330215454, 0.03442753106355667, 0.02753916010260582, 0.08817972242832184, -0.015462561510503292, 0.05122409760951996, 0.0003430015640333295, -0.01798778772354126, 0.020352086052298546, 0.09322929382324219, -0.05006154254078865, 0.046400655061006546, -0.043441664427518845, 0.012250003404915333, -0.05086598917841911, 0.000632967334240675, -0.02909906953573227, -0.0438908115029335, 0.04437175393104553, -0.0064100902527570724, 0.016334785148501396, 0.06977900117635727, 0.04352376610040665, -0.008959362283349037, 0.0030630086548626423, -0.03147897496819496, 0.0334748774766922, -0.04837929829955101, -0.004705514758825302, -0.004161679185926914, -0.08050394058227539, -0.02144329994916916, 0.034824639558792114, -0.003403794253244996, 0.10653313994407654, -0.013146600686013699, -0.017849180847406387, -0.011511427350342274, 0.0472860261797905, -0.07780762016773224, 0.025587381795048714, 0.024715103209018707, -0.04360129311680794, 0.023186136037111282, -0.03573177754878998, -0.03228802606463432, -0.01966124027967453], "cos_apical_negg": [0.5330359935760498, 0.06855221092700958, 0.05162184685468674, 0.08716437220573425, -0.008329702541232109, -0.008950409479439259, -0.0006544194184243679, 0.06687228381633759, -0.026471810415387154, 0.034034427255392075, -0.0335475318133831, 0.01642283797264099, -0.007982833310961723, 0.0006792694330215454, 0.03442753106355667, 0.02753916010260582, 0.08817972242832184, -0.015462561510503292, 0.05122409760951996, 0.0003430015640333295, -0.01798778772354126, 0.020352086052298546, 0.09322929382324219, -0.05006154254078865, 0.046400655061006546, -0.043441664427518845, 0.012250003404915333, -0.05086598917841911, 0.000632967334240675, -0.02909906953573227, -0.0438908115029335, 0.04437175393104553, -0.0064100902527570724, 0.016334785148501396, 0.06977900117635727, 0.04352376610040665, -0.008959362283349037, 0.0030630086548626423, -0.03147897496819496, 0.0334748774766922, -0.04837929829955101, -0.004705514758825302, -0.004161679185926914, -0.08050394058227539, -0.02144329994916916, 0.034824639558792114, -0.003403794253244996, 0.10653313994407654, -0.013146600686013699, -0.017849180847406387, -0.011511427350342274, 0.0472860261797905, -0.07780762016773224, 0.025587381795048714, 0.024715103209018707, -0.04360129311680794, 0.023186136037111282, -0.03573177754878998, -0.03228802606463432, -0.01966124027967453], "cos_Ac_negg": [0.5330359935760498, 0.06855221092700958, 0.05162184685468674, 0.08716437220573425, -0.008329702541232109, -0.008950409479439259, -0.0006544194184243679, 0.06687228381633759, -0.026471810415387154, 0.034034427255392075, -0.0335475318133831, 0.01642283797264099, -0.007982833310961723, 0.0006792694330215454, 0.03442753106355667, 0.02753916010260582, 0.08817972242832184, -0.015462561510503292, 0.05122409760951996, 0.0003430015640333295, -0.01798778772354126, 0.020352086052298546, 0.09322929382324219, -0.05006154254078865, 0.046400655061006546, -0.043441664427518845, 0.012250003404915333, -0.05086598917841911, 0.000632967334240675, -0.02909906953573227, -0.0438908115029335, 0.04437175393104553, -0.0064100902527570724, 0.016334785148501396, 0.06977900117635727, 0.04352376610040665, -0.008959362283349037, 0.0030630086548626423, -0.03147897496819496, 0.0334748774766922, -0.04837929829955101, -0.004705514758825302, -0.004161679185926914, -0.08050394058227539, -0.02144329994916916, 0.034824639558792114, -0.003403794253244996, 0.10653313994407654, -0.013146600686013699, -0.017849180847406387, -0.011511427350342274, 0.0472860261797905, -0.07780762016773224, 0.025587381795048714, 0.024715103209018707, -0.04360129311680794, 0.023186136037111282, -0.03573177754878998, -0.03228802606463432, -0.01966124027967453], "r_norm": [2.28084659576416, 2.210508346557617, 2.094848155975342, 2.1582329273223877, 2.174983024597168, 2.282240390777588, 2.111687183380127, 2.2739200592041016, 2.193063259124756, 2.187309741973877, 2.211914300918579, 2.0239925384521484, 2.1029419898986816, 2.253495931625366, 2.1822845935821533, 2.1494884490966797, 2.254767656326294, 2.07220196723938, 2.197070360183716, 2.2609262466430664, 2.3044190406799316, 2.2762417793273926, 2.2087888717651367, 2.319795608520508, 2.1780340671539307, 2.188530921936035, 2.1326656341552734, 2.1076018810272217, 2.1690168380737305, 2.293008327484131, 2.272796630859375, 2.225308418273926, 2.2039496898651123, 2.2128896713256836, 2.1907856464385986, 2.2084856033325195, 2.224245548248291, 2.2065932750701904, 2.2188873291015625, 2.112318515777588, 2.20369291305542, 2.217454433441162, 2.1485559940338135, 2.2177109718322754, 2.0183145999908447, 2.2709712982177734, 2.1260910034179688, 2.154264450073242, 2.274293899536133, 2.25396466255188, 2.248188018798828, 2.1656055450439453, 2.1179428100585938, 2.1589910984039307, 2.0939483642578125, 2.1234588623046875, 2.1836087703704834, 2.1421847343444824, 2.160097599029541, 2.2136032581329346]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3584, "test_loss": 1.8459331665039063}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3665, "test_loss": 1.8127239501953125}], "final": {"test_acc": 0.3665, "test_loss": 1.8127239501953125, "wall_s": 45.33662247657776, "cos_r_negg": [0.5817568302154541, 0.03133082389831543, 0.056726936250925064, 0.05118889361619949, 0.04365791380405426, -0.02235237881541252, -0.03156343474984169, 0.0500517264008522, -0.0153754698112607, 0.003784403670579195, -0.023922741413116455, 0.07468342781066895, 0.015377920120954514, 0.02706257253885269, 0.031397998332977295, 0.008751153945922852, 0.06810237467288971, 0.020275169983506203, 0.055702194571495056, 0.024997921660542488, -0.02272096835076809, -0.041373834013938904, 0.10833829641342163, -0.02272889018058777, 0.07712725549936295, -0.03970428556203842, 0.017694970592856407, -0.04799677059054375, 0.029361164197325706, -0.03279661759734154, -0.0504496768116951, 0.04907789081335068, -0.02243996039032936, -0.008549323305487633, 0.08012141287326813, 0.05440431833267212, 0.009414292871952057, -0.0051546297036111355, -0.08707298338413239, 0.01931481808423996, 0.013495170511305332, 0.030636053532361984, 0.058429569005966187, -0.019716303795576096, -0.02795001119375229, 0.04706968739628792, 0.044296205043792725, 0.09274455904960632, -0.07268492877483368, -0.005128688178956509, 0.00014444207772612572, 0.06421615183353424, -0.09026831388473511, 0.07781516015529633, 0.04582793265581131, -0.0007914099842309952, 0.007341890130192041, 0.002536190440878272, 0.02620118297636509, -0.036541230976581573], "cos_innovation_negg": [0.5817568302154541, 0.03133082389831543, 0.056726936250925064, 0.05118889361619949, 0.04365791380405426, -0.02235237881541252, -0.03156343474984169, 0.0500517264008522, -0.0153754698112607, 0.003784403670579195, -0.023922741413116455, 0.07468342781066895, 0.015377920120954514, 0.02706257253885269, 0.031397998332977295, 0.008751153945922852, 0.06810237467288971, 0.020275169983506203, 0.055702194571495056, 0.024997921660542488, -0.02272096835076809, -0.041373834013938904, 0.10833829641342163, -0.02272889018058777, 0.07712725549936295, -0.03970428556203842, 0.017694970592856407, -0.04799677059054375, 0.029361164197325706, -0.03279661759734154, -0.0504496768116951, 0.04907789081335068, -0.02243996039032936, -0.008549323305487633, 0.08012141287326813, 0.05440431833267212, 0.009414292871952057, -0.0051546297036111355, -0.08707298338413239, 0.01931481808423996, 0.013495170511305332, 0.030636053532361984, 0.058429569005966187, -0.019716303795576096, -0.02795001119375229, 0.04706968739628792, 0.044296205043792725, 0.09274455904960632, -0.07268492877483368, -0.005128688178956509, 0.00014444207772612572, 0.06421615183353424, -0.09026831388473511, 0.07781516015529633, 0.04582793265581131, -0.0007914099842309952, 0.007341890130192041, 0.002536190440878272, 0.02620118297636509, -0.036541230976581573], "cos_apical_negg": [0.5817568302154541, 0.03133082389831543, 0.056726936250925064, 0.05118889361619949, 0.04365791380405426, -0.02235237881541252, -0.03156343474984169, 0.0500517264008522, -0.0153754698112607, 0.003784403670579195, -0.023922741413116455, 0.07468342781066895, 0.015377920120954514, 0.02706257253885269, 0.031397998332977295, 0.008751153945922852, 0.06810237467288971, 0.020275169983506203, 0.055702194571495056, 0.024997921660542488, -0.02272096835076809, -0.041373834013938904, 0.10833829641342163, -0.02272889018058777, 0.07712725549936295, -0.03970428556203842, 0.017694970592856407, -0.04799677059054375, 0.029361164197325706, -0.03279661759734154, -0.0504496768116951, 0.04907789081335068, -0.02243996039032936, -0.008549323305487633, 0.08012141287326813, 0.05440431833267212, 0.009414292871952057, -0.0051546297036111355, -0.08707298338413239, 0.01931481808423996, 0.013495170511305332, 0.030636053532361984, 0.058429569005966187, -0.019716303795576096, -0.02795001119375229, 0.04706968739628792, 0.044296205043792725, 0.09274455904960632, -0.07268492877483368, -0.005128688178956509, 0.00014444207772612572, 0.06421615183353424, -0.09026831388473511, 0.07781516015529633, 0.04582793265581131, -0.0007914099842309952, 0.007341890130192041, 0.002536190440878272, 0.02620118297636509, -0.036541230976581573], "cos_Ac_negg": [0.5817568302154541, 0.03133082389831543, 0.056726936250925064, 0.05118889361619949, 0.04365791380405426, -0.02235237881541252, -0.03156343474984169, 0.0500517264008522, -0.0153754698112607, 0.003784403670579195, -0.023922741413116455, 0.07468342781066895, 0.015377920120954514, 0.02706257253885269, 0.031397998332977295, 0.008751153945922852, 0.06810237467288971, 0.020275169983506203, 0.055702194571495056, 0.024997921660542488, -0.02272096835076809, -0.041373834013938904, 0.10833829641342163, -0.02272889018058777, 0.07712725549936295, -0.03970428556203842, 0.017694970592856407, -0.04799677059054375, 0.029361164197325706, -0.03279661759734154, -0.0504496768116951, 0.04907789081335068, -0.02243996039032936, -0.008549323305487633, 0.08012141287326813, 0.05440431833267212, 0.009414292871952057, -0.0051546297036111355, -0.08707298338413239, 0.01931481808423996, 0.013495170511305332, 0.030636053532361984, 0.058429569005966187, -0.019716303795576096, -0.02795001119375229, 0.04706968739628792, 0.044296205043792725, 0.09274455904960632, -0.07268492877483368, -0.005128688178956509, 0.00014444207772612572, 0.06421615183353424, -0.09026831388473511, 0.07781516015529633, 0.04582793265581131, -0.0007914099842309952, 0.007341890130192041, 0.002536190440878272, 0.02620118297636509, -0.036541230976581573]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_dfa_w64_d60_s4.json b/results/scale_v3_cifar10_dfa_w64_d60_s4.json new file mode 100644 index 0000000..be27cd2 --- /dev/null +++ b/results/scale_v3_cifar10_dfa_w64_d60_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "dfa", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.05, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_dfa_w64_d60_s4", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6918692588806152, "cos_r_negg": [-0.037414051592350006, -0.0028529749251902103, -0.054665934294462204, -0.012028934434056282, -0.029990021139383316, 0.05329199880361557, -0.0034322100691497326, -0.11187580227851868, 0.027348166331648827, 0.03484438359737396, 0.005401864647865295, -0.07433269172906876, -0.008908074349164963, 0.005322254728525877, -0.0026320726610720158, 0.023683061823248863, 0.01821216754615307, -0.030909225344657898, 0.027072368189692497, 0.03808046132326126, -0.036475181579589844, 0.017430506646633148, -0.060206860303878784, 0.05741240829229355, -0.025437306612730026, -0.06906218826770782, -0.04085918515920639, 0.01807219162583351, -0.06371022760868073, 0.016573622822761536, 0.011960912495851517, 0.03196274861693382, 0.06078161299228668, -0.04183672368526459, -0.0401955246925354, 0.02006252110004425, 0.057506345212459564, -0.04063631594181061, -0.014354291372001171, -0.03390126675367355, -0.06117922067642212, 0.037960510700941086, -0.01032742578536272, 0.055927228182554245, -0.03214752674102783, -0.05085998401045799, 0.002562712412327528, 0.02092983014881611, -0.062292709946632385, 0.010097211226820946, 0.023000245913863182, 0.03037109412252903, -0.013356085866689682, 0.03924810513854027, 0.10382106900215149, 0.006241656839847565, 0.09594763815402985, -0.016425050795078278, -0.10900639742612839, 0.0001760171726346016], "cos_innovation_negg": [-0.037414051592350006, -0.0028529749251902103, -0.054665934294462204, -0.012028934434056282, -0.029990021139383316, 0.05329199880361557, -0.0034322100691497326, -0.11187580227851868, 0.027348166331648827, 0.03484438359737396, 0.005401864647865295, -0.07433269172906876, -0.008908074349164963, 0.005322254728525877, -0.0026320726610720158, 0.023683061823248863, 0.01821216754615307, -0.030909225344657898, 0.027072368189692497, 0.03808046132326126, -0.036475181579589844, 0.017430506646633148, -0.060206860303878784, 0.05741240829229355, -0.025437306612730026, -0.06906218826770782, -0.04085918515920639, 0.01807219162583351, -0.06371022760868073, 0.016573622822761536, 0.011960912495851517, 0.03196274861693382, 0.06078161299228668, -0.04183672368526459, -0.0401955246925354, 0.02006252110004425, 0.057506345212459564, -0.04063631594181061, -0.014354291372001171, -0.03390126675367355, -0.06117922067642212, 0.037960510700941086, -0.01032742578536272, 0.055927228182554245, -0.03214752674102783, -0.05085998401045799, 0.002562712412327528, 0.02092983014881611, -0.062292709946632385, 0.010097211226820946, 0.023000245913863182, 0.03037109412252903, -0.013356085866689682, 0.03924810513854027, 0.10382106900215149, 0.006241656839847565, 0.09594763815402985, -0.016425050795078278, -0.10900639742612839, 0.0001760171726346016], "cos_apical_negg": [-0.037414051592350006, -0.0028529749251902103, -0.054665934294462204, -0.012028934434056282, -0.029990021139383316, 0.05329199880361557, -0.0034322100691497326, -0.11187580227851868, 0.027348166331648827, 0.03484438359737396, 0.005401864647865295, -0.07433269172906876, -0.008908074349164963, 0.005322254728525877, -0.0026320726610720158, 0.023683061823248863, 0.01821216754615307, -0.030909225344657898, 0.027072368189692497, 0.03808046132326126, -0.036475181579589844, 0.017430506646633148, -0.060206860303878784, 0.05741240829229355, -0.025437306612730026, -0.06906218826770782, -0.04085918515920639, 0.01807219162583351, -0.06371022760868073, 0.016573622822761536, 0.011960912495851517, 0.03196274861693382, 0.06078161299228668, -0.04183672368526459, -0.0401955246925354, 0.02006252110004425, 0.057506345212459564, -0.04063631594181061, -0.014354291372001171, -0.03390126675367355, -0.06117922067642212, 0.037960510700941086, -0.01032742578536272, 0.055927228182554245, -0.03214752674102783, -0.05085998401045799, 0.002562712412327528, 0.02092983014881611, -0.062292709946632385, 0.010097211226820946, 0.023000245913863182, 0.03037109412252903, -0.013356085866689682, 0.03924810513854027, 0.10382106900215149, 0.006241656839847565, 0.09594763815402985, -0.016425050795078278, -0.10900639742612839, 0.0001760171726346016], "cos_Ac_negg": [-0.037414051592350006, -0.0028529749251902103, -0.054665934294462204, -0.012028934434056282, -0.029990021139383316, 0.05329199880361557, -0.0034322100691497326, -0.11187580227851868, 0.027348166331648827, 0.03484438359737396, 0.005401864647865295, -0.07433269172906876, -0.008908074349164963, 0.005322254728525877, -0.0026320726610720158, 0.023683061823248863, 0.01821216754615307, -0.030909225344657898, 0.027072368189692497, 0.03808046132326126, -0.036475181579589844, 0.017430506646633148, -0.060206860303878784, 0.05741240829229355, -0.025437306612730026, -0.06906218826770782, -0.04085918515920639, 0.01807219162583351, -0.06371022760868073, 0.016573622822761536, 0.011960912495851517, 0.03196274861693382, 0.06078161299228668, -0.04183672368526459, -0.0401955246925354, 0.02006252110004425, 0.057506345212459564, -0.04063631594181061, -0.014354291372001171, -0.03390126675367355, -0.06117922067642212, 0.037960510700941086, -0.01032742578536272, 0.055927228182554245, -0.03214752674102783, -0.05085998401045799, 0.002562712412327528, 0.02092983014881611, -0.062292709946632385, 0.010097211226820946, 0.023000245913863182, 0.03037109412252903, -0.013356085866689682, 0.03924810513854027, 0.10382106900215149, 0.006241656839847565, 0.09594763815402985, -0.016425050795078278, -0.10900639742612839, 0.0001760171726346016], "r_norm": [2.5585856437683105, 2.5181503295898438, 2.5646228790283203, 2.3973093032836914, 2.425574541091919, 2.470545768737793, 2.44059157371521, 2.434709072113037, 2.4547786712646484, 2.48994517326355, 2.4773926734924316, 2.4522080421447754, 2.443126678466797, 2.3443167209625244, 2.3908872604370117, 2.5021166801452637, 2.580672264099121, 2.4316048622131348, 2.468903064727783, 2.5342869758605957, 2.5060877799987793, 2.593197822570801, 2.3915252685546875, 2.5748143196105957, 2.364664077758789, 2.4329833984375, 2.3553965091705322, 2.421529769897461, 2.5368752479553223, 2.558291435241699, 2.361297130584717, 2.395977020263672, 2.470844268798828, 2.5835165977478027, 2.32879900932312, 2.4639298915863037, 2.6483755111694336, 2.455132484436035, 2.548173666000366, 2.4606847763061523, 2.4713215827941895, 2.4578258991241455, 2.5337564945220947, 2.380887269973755, 2.421393871307373, 2.4823501110076904, 2.4976062774658203, 2.459674835205078, 2.360995292663574, 2.485690116882324, 2.326770544052124, 2.457512378692627, 2.4043235778808594, 2.3826351165771484, 2.383045196533203, 2.4985876083374023, 2.529938220977783, 2.463041305541992, 2.5037412643432617, 2.367680788040161]}, {"epoch_end": 0, "step": 391, "test_acc": 0.347, "test_loss": 1.8751915161132813}, {"step": 500, "epoch": 1, "train_loss": 1.8213450908660889, "cos_r_negg": [0.5531076788902283, 0.010648340918123722, 0.05643956735730171, 0.026077724993228912, 0.01408854778856039, -0.004972496069967747, -0.04337833449244499, 0.06296228617429733, 0.04684184119105339, -0.07821905612945557, -0.009295742027461529, 0.005955658387392759, -0.04401475563645363, 0.03754936903715134, -0.03872787952423096, 0.03463492542505264, -0.012126315385103226, -0.02981529012322426, -0.013925951905548573, -0.04895723983645439, 0.02408735826611519, -0.0037605210673063993, -0.004745983984321356, -0.011288284324109554, -0.02553575485944748, 0.030575040727853775, 0.0053910487331449986, -0.012438648380339146, -0.05085151642560959, -0.11037540435791016, -0.041079506278038025, 0.08036839216947556, -0.037528786808252335, -0.03934481739997864, 0.0025282700080424547, 0.05720853805541992, 0.029822615906596184, 8.558575063943863e-05, -0.016070079058408737, 0.024425091221928596, -0.08186234533786774, 0.08049727976322174, -0.044807836413383484, 0.029020026326179504, -0.018535487353801727, 0.004029462113976479, 0.03768383711576462, 0.003076717024669051, -0.012039786204695702, 0.037689950317144394, 0.007063157390803099, -0.0169571191072464, -0.010830679908394814, -0.07586954534053802, -0.0132209537550807, 0.07407307624816895, 0.059688977897167206, -0.020425468683242798, -0.05862516909837723, 0.03696390986442566], "cos_innovation_negg": [0.5531076788902283, 0.010648340918123722, 0.05643956735730171, 0.026077724993228912, 0.01408854778856039, -0.004972496069967747, -0.04337833449244499, 0.06296228617429733, 0.04684184119105339, -0.07821905612945557, -0.009295742027461529, 0.005955658387392759, -0.04401475563645363, 0.03754936903715134, -0.03872787952423096, 0.03463492542505264, -0.012126315385103226, -0.02981529012322426, -0.013925951905548573, -0.04895723983645439, 0.02408735826611519, -0.0037605210673063993, -0.004745983984321356, -0.011288284324109554, -0.02553575485944748, 0.030575040727853775, 0.0053910487331449986, -0.012438648380339146, -0.05085151642560959, -0.11037540435791016, -0.041079506278038025, 0.08036839216947556, -0.037528786808252335, -0.03934481739997864, 0.0025282700080424547, 0.05720853805541992, 0.029822615906596184, 8.558575063943863e-05, -0.016070079058408737, 0.024425091221928596, -0.08186234533786774, 0.08049727976322174, -0.044807836413383484, 0.029020026326179504, -0.018535487353801727, 0.004029462113976479, 0.03768383711576462, 0.003076717024669051, -0.012039786204695702, 0.037689950317144394, 0.007063157390803099, -0.0169571191072464, -0.010830679908394814, -0.07586954534053802, -0.0132209537550807, 0.07407307624816895, 0.059688977897167206, -0.020425468683242798, -0.05862516909837723, 0.03696390986442566], "cos_apical_negg": [0.5531076788902283, 0.010648340918123722, 0.05643956735730171, 0.026077724993228912, 0.01408854778856039, -0.004972496069967747, -0.04337833449244499, 0.06296228617429733, 0.04684184119105339, -0.07821905612945557, -0.009295742027461529, 0.005955658387392759, -0.04401475563645363, 0.03754936903715134, -0.03872787952423096, 0.03463492542505264, -0.012126315385103226, -0.02981529012322426, -0.013925951905548573, -0.04895723983645439, 0.02408735826611519, -0.0037605210673063993, -0.004745983984321356, -0.011288284324109554, -0.02553575485944748, 0.030575040727853775, 0.0053910487331449986, -0.012438648380339146, -0.05085151642560959, -0.11037540435791016, -0.041079506278038025, 0.08036839216947556, -0.037528786808252335, -0.03934481739997864, 0.0025282700080424547, 0.05720853805541992, 0.029822615906596184, 8.558575063943863e-05, -0.016070079058408737, 0.024425091221928596, -0.08186234533786774, 0.08049727976322174, -0.044807836413383484, 0.029020026326179504, -0.018535487353801727, 0.004029462113976479, 0.03768383711576462, 0.003076717024669051, -0.012039786204695702, 0.037689950317144394, 0.007063157390803099, -0.0169571191072464, -0.010830679908394814, -0.07586954534053802, -0.0132209537550807, 0.07407307624816895, 0.059688977897167206, -0.020425468683242798, -0.05862516909837723, 0.03696390986442566], "cos_Ac_negg": [0.5531076788902283, 0.010648340918123722, 0.05643956735730171, 0.026077724993228912, 0.01408854778856039, -0.004972496069967747, -0.04337833449244499, 0.06296228617429733, 0.04684184119105339, -0.07821905612945557, -0.009295742027461529, 0.005955658387392759, -0.04401475563645363, 0.03754936903715134, -0.03872787952423096, 0.03463492542505264, -0.012126315385103226, -0.02981529012322426, -0.013925951905548573, -0.04895723983645439, 0.02408735826611519, -0.0037605210673063993, -0.004745983984321356, -0.011288284324109554, -0.02553575485944748, 0.030575040727853775, 0.0053910487331449986, -0.012438648380339146, -0.05085151642560959, -0.11037540435791016, -0.041079506278038025, 0.08036839216947556, -0.037528786808252335, -0.03934481739997864, 0.0025282700080424547, 0.05720853805541992, 0.029822615906596184, 8.558575063943863e-05, -0.016070079058408737, 0.024425091221928596, -0.08186234533786774, 0.08049727976322174, -0.044807836413383484, 0.029020026326179504, -0.018535487353801727, 0.004029462113976479, 0.03768383711576462, 0.003076717024669051, -0.012039786204695702, 0.037689950317144394, 0.007063157390803099, -0.0169571191072464, -0.010830679908394814, -0.07586954534053802, -0.0132209537550807, 0.07407307624816895, 0.059688977897167206, -0.020425468683242798, -0.05862516909837723, 0.03696390986442566], "r_norm": [2.2668089866638184, 2.2286651134490967, 2.271970272064209, 2.137986660003662, 2.1760787963867188, 2.2030792236328125, 2.1382670402526855, 2.124001979827881, 2.191345691680908, 2.2414255142211914, 2.206237554550171, 2.141871690750122, 2.1780307292938232, 2.0781381130218506, 2.1217825412750244, 2.2277321815490723, 2.302032470703125, 2.1094799041748047, 2.152195453643799, 2.2596535682678223, 2.209888458251953, 2.2921009063720703, 2.1111698150634766, 2.250282049179077, 2.1164474487304688, 2.1762876510620117, 2.087702751159668, 2.136573076248169, 2.2555909156799316, 2.272592544555664, 2.124788999557495, 2.126204252243042, 2.2049756050109863, 2.2845869064331055, 2.0533406734466553, 2.1725053787231445, 2.3358402252197266, 2.1688904762268066, 2.2532334327697754, 2.1950769424438477, 2.1407790184020996, 2.1844682693481445, 2.244250774383545, 2.114405632019043, 2.145517349243164, 2.2021920680999756, 2.2008676528930664, 2.160367488861084, 2.0971405506134033, 2.2253780364990234, 2.0831708908081055, 2.17439866065979, 2.1249170303344727, 2.1198954582214355, 2.1070141792297363, 2.2050018310546875, 2.226027011871338, 2.174729824066162, 2.1999001502990723, 2.1154282093048096]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3589, "test_loss": 1.8626206420898437}, {"step": 1000, "epoch": 2, "train_loss": 1.8082916736602783, "cos_r_negg": [0.5772711038589478, 0.022301051765680313, 0.032155841588974, 0.07773202657699585, -0.02474537119269371, 0.04731186851859093, -0.024519361555576324, 0.036791980266571045, 0.06838394701480865, -0.08703818917274475, -0.016849961131811142, 0.007404455449432135, 0.03726254403591156, 0.07393119484186172, -0.06879924237728119, 0.053731247782707214, -0.04143936559557915, -0.006858188658952713, -0.019606173038482666, 0.016490813344717026, -0.016047628596425056, -0.024410627782344818, -0.024281444028019905, -0.05238782614469528, -0.005266563966870308, 0.015507731586694717, 0.0781889408826828, -0.055368952453136444, -0.009982608258724213, -0.04081203415989876, -0.033765099942684174, 0.012983039952814579, -0.026144884526729584, -0.03163754567503929, -0.008402347564697266, 0.08629274368286133, 0.024958964437246323, 0.039797987788915634, 0.0374150425195694, -0.017730850726366043, -0.06318195909261703, 0.07919758558273315, -0.041827570647001266, 0.047349877655506134, 0.0541478730738163, 0.0026697958819568157, 0.0757795125246048, -0.005099573638290167, -0.03752800077199936, 0.06168989837169647, 0.01624457910656929, -0.01865878328680992, -0.02403246983885765, -0.03217144310474396, -0.03725310042500496, 0.02615518681704998, -0.026163369417190552, -0.004356469959020615, -0.04227064549922943, -0.034727491438388824], "cos_innovation_negg": [0.5772711038589478, 0.022301051765680313, 0.032155841588974, 0.07773202657699585, -0.02474537119269371, 0.04731186851859093, -0.024519361555576324, 0.036791980266571045, 0.06838394701480865, -0.08703818917274475, -0.016849961131811142, 0.007404455449432135, 0.03726254403591156, 0.07393119484186172, -0.06879924237728119, 0.053731247782707214, -0.04143936559557915, -0.006858188658952713, -0.019606173038482666, 0.016490813344717026, -0.016047628596425056, -0.024410627782344818, -0.024281444028019905, -0.05238782614469528, -0.005266563966870308, 0.015507731586694717, 0.0781889408826828, -0.055368952453136444, -0.009982608258724213, -0.04081203415989876, -0.033765099942684174, 0.012983039952814579, -0.026144884526729584, -0.03163754567503929, -0.008402347564697266, 0.08629274368286133, 0.024958964437246323, 0.039797987788915634, 0.0374150425195694, -0.017730850726366043, -0.06318195909261703, 0.07919758558273315, -0.041827570647001266, 0.047349877655506134, 0.0541478730738163, 0.0026697958819568157, 0.0757795125246048, -0.005099573638290167, -0.03752800077199936, 0.06168989837169647, 0.01624457910656929, -0.01865878328680992, -0.02403246983885765, -0.03217144310474396, -0.03725310042500496, 0.02615518681704998, -0.026163369417190552, -0.004356469959020615, -0.04227064549922943, -0.034727491438388824], "cos_apical_negg": [0.5772711038589478, 0.022301051765680313, 0.032155841588974, 0.07773202657699585, -0.02474537119269371, 0.04731186851859093, -0.024519361555576324, 0.036791980266571045, 0.06838394701480865, -0.08703818917274475, -0.016849961131811142, 0.007404455449432135, 0.03726254403591156, 0.07393119484186172, -0.06879924237728119, 0.053731247782707214, -0.04143936559557915, -0.006858188658952713, -0.019606173038482666, 0.016490813344717026, -0.016047628596425056, -0.024410627782344818, -0.024281444028019905, -0.05238782614469528, -0.005266563966870308, 0.015507731586694717, 0.0781889408826828, -0.055368952453136444, -0.009982608258724213, -0.04081203415989876, -0.033765099942684174, 0.012983039952814579, -0.026144884526729584, -0.03163754567503929, -0.008402347564697266, 0.08629274368286133, 0.024958964437246323, 0.039797987788915634, 0.0374150425195694, -0.017730850726366043, -0.06318195909261703, 0.07919758558273315, -0.041827570647001266, 0.047349877655506134, 0.0541478730738163, 0.0026697958819568157, 0.0757795125246048, -0.005099573638290167, -0.03752800077199936, 0.06168989837169647, 0.01624457910656929, -0.01865878328680992, -0.02403246983885765, -0.03217144310474396, -0.03725310042500496, 0.02615518681704998, -0.026163369417190552, -0.004356469959020615, -0.04227064549922943, -0.034727491438388824], "cos_Ac_negg": [0.5772711038589478, 0.022301051765680313, 0.032155841588974, 0.07773202657699585, -0.02474537119269371, 0.04731186851859093, -0.024519361555576324, 0.036791980266571045, 0.06838394701480865, -0.08703818917274475, -0.016849961131811142, 0.007404455449432135, 0.03726254403591156, 0.07393119484186172, -0.06879924237728119, 0.053731247782707214, -0.04143936559557915, -0.006858188658952713, -0.019606173038482666, 0.016490813344717026, -0.016047628596425056, -0.024410627782344818, -0.024281444028019905, -0.05238782614469528, -0.005266563966870308, 0.015507731586694717, 0.0781889408826828, -0.055368952453136444, -0.009982608258724213, -0.04081203415989876, -0.033765099942684174, 0.012983039952814579, -0.026144884526729584, -0.03163754567503929, -0.008402347564697266, 0.08629274368286133, 0.024958964437246323, 0.039797987788915634, 0.0374150425195694, -0.017730850726366043, -0.06318195909261703, 0.07919758558273315, -0.041827570647001266, 0.047349877655506134, 0.0541478730738163, 0.0026697958819568157, 0.0757795125246048, -0.005099573638290167, -0.03752800077199936, 0.06168989837169647, 0.01624457910656929, -0.01865878328680992, -0.02403246983885765, -0.03217144310474396, -0.03725310042500496, 0.02615518681704998, -0.026163369417190552, -0.004356469959020615, -0.04227064549922943, -0.034727491438388824], "r_norm": [2.2569527626037598, 2.213512897491455, 2.262234687805176, 2.133873462677002, 2.1648001670837402, 2.1931161880493164, 2.1322126388549805, 2.113229274749756, 2.1879642009735107, 2.2315547466278076, 2.190305471420288, 2.138906478881836, 2.166815757751465, 2.06341290473938, 2.1164820194244385, 2.229915142059326, 2.29317569732666, 2.0981292724609375, 2.1424736976623535, 2.2526912689208984, 2.203888416290283, 2.286067008972168, 2.0997042655944824, 2.25266432762146, 2.1137876510620117, 2.1620516777038574, 2.086796760559082, 2.1146671772003174, 2.251715660095215, 2.2569937705993652, 2.1237001419067383, 2.114114284515381, 2.201896905899048, 2.283658266067505, 2.0540478229522705, 2.171139717102051, 2.332195281982422, 2.164245128631592, 2.2440543174743652, 2.202272415161133, 2.132251739501953, 2.177074432373047, 2.224301815032959, 2.100316047668457, 2.1419622898101807, 2.1810121536254883, 2.195061206817627, 2.149263858795166, 2.0860493183135986, 2.2124271392822266, 2.0833592414855957, 2.1500306129455566, 2.1120290756225586, 2.107145309448242, 2.0901613235473633, 2.1897993087768555, 2.212405204772949, 2.1706151962280273, 2.196743965148926, 2.1062331199645996]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3567, "test_loss": 1.8509368530273438}, {"step": 1500, "epoch": 3, "train_loss": 1.6837269067764282, "cos_r_negg": [0.5648865699768066, 0.06946022808551788, 0.0689295306801796, 0.016047975048422813, 0.027184724807739258, -0.0037121777422726154, -0.00188345555216074, -0.014041886664927006, 0.010996182449162006, 0.0013104379177093506, 0.010440901853144169, -0.04100625216960907, -0.014227133244276047, 0.007865268737077713, -0.08391951769590378, 0.02313748188316822, -0.013638537377119064, 0.0380057692527771, -0.009726918302476406, -0.017707467079162598, 1.2996839359402657e-05, 0.0327940508723259, 0.011521142907440662, -0.030674725770950317, -0.009084416553378105, 0.053915418684482574, 0.044526055455207825, -0.08661733567714691, -0.08987803012132645, -0.02304910309612751, 0.030008945614099503, 0.003957095555961132, 0.0034250395838171244, -0.01661277748644352, 0.014571542851626873, 0.1101943701505661, 0.04530530795454979, 0.0332380086183548, 0.051617786288261414, 0.0423697903752327, -0.03682206943631172, 0.06059061363339424, -0.044939134269952774, -0.0018963469192385674, -0.0023223580792546272, 0.03827180713415146, 0.0635790303349495, -0.03181951493024826, -0.06334107369184494, 0.07252269238233566, -0.012763847596943378, 0.022637145593762398, 0.04536272585391998, -0.012558459304273129, -0.007688129786401987, 0.10340453684329987, 0.01658482477068901, -0.00919765792787075, 0.006536683067679405, 0.058168571442365646], "cos_innovation_negg": [0.5648865699768066, 0.06946022808551788, 0.0689295306801796, 0.016047975048422813, 0.027184724807739258, -0.0037121777422726154, -0.00188345555216074, -0.014041886664927006, 0.010996182449162006, 0.0013104379177093506, 0.010440901853144169, -0.04100625216960907, -0.014227133244276047, 0.007865268737077713, -0.08391951769590378, 0.02313748188316822, -0.013638537377119064, 0.0380057692527771, -0.009726918302476406, -0.017707467079162598, 1.2996839359402657e-05, 0.0327940508723259, 0.011521142907440662, -0.030674725770950317, -0.009084416553378105, 0.053915418684482574, 0.044526055455207825, -0.08661733567714691, -0.08987803012132645, -0.02304910309612751, 0.030008945614099503, 0.003957095555961132, 0.0034250395838171244, -0.01661277748644352, 0.014571542851626873, 0.1101943701505661, 0.04530530795454979, 0.0332380086183548, 0.051617786288261414, 0.0423697903752327, -0.03682206943631172, 0.06059061363339424, -0.044939134269952774, -0.0018963469192385674, -0.0023223580792546272, 0.03827180713415146, 0.0635790303349495, -0.03181951493024826, -0.06334107369184494, 0.07252269238233566, -0.012763847596943378, 0.022637145593762398, 0.04536272585391998, -0.012558459304273129, -0.007688129786401987, 0.10340453684329987, 0.01658482477068901, -0.00919765792787075, 0.006536683067679405, 0.058168571442365646], "cos_apical_negg": [0.5648865699768066, 0.06946022808551788, 0.0689295306801796, 0.016047975048422813, 0.027184724807739258, -0.0037121777422726154, -0.00188345555216074, -0.014041886664927006, 0.010996182449162006, 0.0013104379177093506, 0.010440901853144169, -0.04100625216960907, -0.014227133244276047, 0.007865268737077713, -0.08391951769590378, 0.02313748188316822, -0.013638537377119064, 0.0380057692527771, -0.009726918302476406, -0.017707467079162598, 1.2996839359402657e-05, 0.0327940508723259, 0.011521142907440662, -0.030674725770950317, -0.009084416553378105, 0.053915418684482574, 0.044526055455207825, -0.08661733567714691, -0.08987803012132645, -0.02304910309612751, 0.030008945614099503, 0.003957095555961132, 0.0034250395838171244, -0.01661277748644352, 0.014571542851626873, 0.1101943701505661, 0.04530530795454979, 0.0332380086183548, 0.051617786288261414, 0.0423697903752327, -0.03682206943631172, 0.06059061363339424, -0.044939134269952774, -0.0018963469192385674, -0.0023223580792546272, 0.03827180713415146, 0.0635790303349495, -0.03181951493024826, -0.06334107369184494, 0.07252269238233566, -0.012763847596943378, 0.022637145593762398, 0.04536272585391998, -0.012558459304273129, -0.007688129786401987, 0.10340453684329987, 0.01658482477068901, -0.00919765792787075, 0.006536683067679405, 0.058168571442365646], "cos_Ac_negg": [0.5648865699768066, 0.06946022808551788, 0.0689295306801796, 0.016047975048422813, 0.027184724807739258, -0.0037121777422726154, -0.00188345555216074, -0.014041886664927006, 0.010996182449162006, 0.0013104379177093506, 0.010440901853144169, -0.04100625216960907, -0.014227133244276047, 0.007865268737077713, -0.08391951769590378, 0.02313748188316822, -0.013638537377119064, 0.0380057692527771, -0.009726918302476406, -0.017707467079162598, 1.2996839359402657e-05, 0.0327940508723259, 0.011521142907440662, -0.030674725770950317, -0.009084416553378105, 0.053915418684482574, 0.044526055455207825, -0.08661733567714691, -0.08987803012132645, -0.02304910309612751, 0.030008945614099503, 0.003957095555961132, 0.0034250395838171244, -0.01661277748644352, 0.014571542851626873, 0.1101943701505661, 0.04530530795454979, 0.0332380086183548, 0.051617786288261414, 0.0423697903752327, -0.03682206943631172, 0.06059061363339424, -0.044939134269952774, -0.0018963469192385674, -0.0023223580792546272, 0.03827180713415146, 0.0635790303349495, -0.03181951493024826, -0.06334107369184494, 0.07252269238233566, -0.012763847596943378, 0.022637145593762398, 0.04536272585391998, -0.012558459304273129, -0.007688129786401987, 0.10340453684329987, 0.01658482477068901, -0.00919765792787075, 0.006536683067679405, 0.058168571442365646], "r_norm": [2.2321786880493164, 2.196965217590332, 2.2361550331115723, 2.1093597412109375, 2.142275810241699, 2.1748976707458496, 2.1086888313293457, 2.090163230895996, 2.1619715690612793, 2.194758176803589, 2.1661875247955322, 2.1052215099334717, 2.1355338096618652, 2.046527862548828, 2.0936834812164307, 2.1927523612976074, 2.263762950897217, 2.0665082931518555, 2.1194331645965576, 2.230807304382324, 2.177485466003418, 2.2552690505981445, 2.0730667114257812, 2.2154641151428223, 2.088258981704712, 2.1344759464263916, 2.0561013221740723, 2.0958547592163086, 2.218369245529175, 2.2283055782318115, 2.099677324295044, 2.0915074348449707, 2.174365282058716, 2.245183229446411, 2.0259180068969727, 2.1336750984191895, 2.3084373474121094, 2.140641689300537, 2.2120208740234375, 2.1657462120056152, 2.1040573120117188, 2.151247501373291, 2.2000958919525146, 2.0798468589782715, 2.1148722171783447, 2.1535186767578125, 2.175259590148926, 2.1279540061950684, 2.065502643585205, 2.1884865760803223, 2.058825731277466, 2.126791000366211, 2.095001697540283, 2.094183921813965, 2.0786945819854736, 2.1680731773376465, 2.185600996017456, 2.1513381004333496, 2.16831636428833, 2.0839295387268066]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3675, "test_loss": 1.816008349609375}, {"epoch_end": 4, "step": 1955, "test_acc": 0.3742, "test_loss": 1.8041421142578125}], "final": {"test_acc": 0.3742, "test_loss": 1.8041421142578125, "wall_s": 44.22455668449402, "cos_r_negg": [0.596982479095459, 0.05136440321803093, 0.08694332838058472, 0.025471724569797516, -0.038266900926828384, -0.019973063841462135, -0.048480987548828125, 0.043332137167453766, 0.07160402834415436, 0.0031838256400078535, 0.024332629516720772, 0.023919688537716866, 0.013141728937625885, 0.03220628947019577, -0.06388810276985168, 0.056075140833854675, 0.005829954519867897, -0.007993511855602264, 0.02282010018825531, -0.03376900404691696, 0.010672175325453281, 0.009448038414120674, 0.01923620142042637, -0.030952278524637222, -0.014296494424343109, 0.06497082114219666, 0.03363164886832237, -0.07824541628360748, -0.07161377370357513, 0.024450957775115967, 0.01999933458864689, 0.04040498286485672, -0.027892522513866425, -0.0026604291051626205, 0.015942219644784927, 0.05475258827209473, 0.08153602480888367, 0.04304179549217224, -0.011818019673228264, 0.04176750034093857, -0.060032933950424194, 0.06945739686489105, -0.028158754110336304, -0.013861177489161491, 0.028401972725987434, -0.003857881296426058, 0.07817532122135162, -0.07094581425189972, -0.06632339954376221, 0.05141926556825638, 0.05274932086467743, 0.018980547785758972, 0.013638835400342941, 0.017520654946565628, -0.032574281096458435, 0.07443814724683762, 0.0825805813074112, 0.003304620273411274, -0.04386940598487854, 0.07008638232946396], "cos_innovation_negg": [0.596982479095459, 0.05136440321803093, 0.08694332838058472, 0.025471724569797516, -0.038266900926828384, -0.019973063841462135, -0.048480987548828125, 0.043332137167453766, 0.07160402834415436, 0.0031838256400078535, 0.024332629516720772, 0.023919688537716866, 0.013141728937625885, 0.03220628947019577, -0.06388810276985168, 0.056075140833854675, 0.005829954519867897, -0.007993511855602264, 0.02282010018825531, -0.03376900404691696, 0.010672175325453281, 0.009448038414120674, 0.01923620142042637, -0.030952278524637222, -0.014296494424343109, 0.06497082114219666, 0.03363164886832237, -0.07824541628360748, -0.07161377370357513, 0.024450957775115967, 0.01999933458864689, 0.04040498286485672, -0.027892522513866425, -0.0026604291051626205, 0.015942219644784927, 0.05475258827209473, 0.08153602480888367, 0.04304179549217224, -0.011818019673228264, 0.04176750034093857, -0.060032933950424194, 0.06945739686489105, -0.028158754110336304, -0.013861177489161491, 0.028401972725987434, -0.003857881296426058, 0.07817532122135162, -0.07094581425189972, -0.06632339954376221, 0.05141926556825638, 0.05274932086467743, 0.018980547785758972, 0.013638835400342941, 0.017520654946565628, -0.032574281096458435, 0.07443814724683762, 0.0825805813074112, 0.003304620273411274, -0.04386940598487854, 0.07008638232946396], "cos_apical_negg": [0.596982479095459, 0.05136440321803093, 0.08694332838058472, 0.025471724569797516, -0.038266900926828384, -0.019973063841462135, -0.048480987548828125, 0.043332137167453766, 0.07160402834415436, 0.0031838256400078535, 0.024332629516720772, 0.023919688537716866, 0.013141728937625885, 0.03220628947019577, -0.06388810276985168, 0.056075140833854675, 0.005829954519867897, -0.007993511855602264, 0.02282010018825531, -0.03376900404691696, 0.010672175325453281, 0.009448038414120674, 0.01923620142042637, -0.030952278524637222, -0.014296494424343109, 0.06497082114219666, 0.03363164886832237, -0.07824541628360748, -0.07161377370357513, 0.024450957775115967, 0.01999933458864689, 0.04040498286485672, -0.027892522513866425, -0.0026604291051626205, 0.015942219644784927, 0.05475258827209473, 0.08153602480888367, 0.04304179549217224, -0.011818019673228264, 0.04176750034093857, -0.060032933950424194, 0.06945739686489105, -0.028158754110336304, -0.013861177489161491, 0.028401972725987434, -0.003857881296426058, 0.07817532122135162, -0.07094581425189972, -0.06632339954376221, 0.05141926556825638, 0.05274932086467743, 0.018980547785758972, 0.013638835400342941, 0.017520654946565628, -0.032574281096458435, 0.07443814724683762, 0.0825805813074112, 0.003304620273411274, -0.04386940598487854, 0.07008638232946396], "cos_Ac_negg": [0.596982479095459, 0.05136440321803093, 0.08694332838058472, 0.025471724569797516, -0.038266900926828384, -0.019973063841462135, -0.048480987548828125, 0.043332137167453766, 0.07160402834415436, 0.0031838256400078535, 0.024332629516720772, 0.023919688537716866, 0.013141728937625885, 0.03220628947019577, -0.06388810276985168, 0.056075140833854675, 0.005829954519867897, -0.007993511855602264, 0.02282010018825531, -0.03376900404691696, 0.010672175325453281, 0.009448038414120674, 0.01923620142042637, -0.030952278524637222, -0.014296494424343109, 0.06497082114219666, 0.03363164886832237, -0.07824541628360748, -0.07161377370357513, 0.024450957775115967, 0.01999933458864689, 0.04040498286485672, -0.027892522513866425, -0.0026604291051626205, 0.015942219644784927, 0.05475258827209473, 0.08153602480888367, 0.04304179549217224, -0.011818019673228264, 0.04176750034093857, -0.060032933950424194, 0.06945739686489105, -0.028158754110336304, -0.013861177489161491, 0.028401972725987434, -0.003857881296426058, 0.07817532122135162, -0.07094581425189972, -0.06632339954376221, 0.05141926556825638, 0.05274932086467743, 0.018980547785758972, 0.013638835400342941, 0.017520654946565628, -0.032574281096458435, 0.07443814724683762, 0.0825805813074112, 0.003304620273411274, -0.04386940598487854, 0.07008638232946396]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d30_s0.json b/results/scale_v3_cifar10_sdil_w64_d30_s0.json new file mode 100644 index 0000000..6a2a758 --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d30_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d30_s0", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.4157285690307617, "cos_r_negg": [0.007712585851550102, -0.03975548595190048, 0.025203227996826172, 0.03284638747572899, -0.02114245295524597, -0.02241712063550949, 0.01855619251728058, -0.011118985712528229, 0.09588839113712311, -0.023850148543715477, -0.046825066208839417, 0.003365863114595413, 0.0047249626368284225, 0.023461898788809776, -0.01765047386288643, 0.03156798332929611, -0.01771402545273304, 0.004477180540561676, 0.0037541198544204235, 0.035115797072649, 0.04352542757987976, -0.009780683554708958, 0.034562088549137115, -0.032053593546152115, -0.05173061788082123, -0.044041719287633896, 0.01968034915626049, 0.007763845846056938, 0.05400868505239487, 0.04457078129053116], "cos_innovation_negg": [0.007712585851550102, -0.03975548595190048, 0.025203227996826172, 0.03284638747572899, -0.02114245295524597, -0.02241712063550949, 0.01855619251728058, -0.011118985712528229, 0.09588839113712311, -0.023850148543715477, -0.046825066208839417, 0.003365863114595413, 0.0047249626368284225, 0.023461898788809776, -0.01765047386288643, 0.03156798332929611, -0.01771402545273304, 0.004477180540561676, 0.0037541198544204235, 0.035115797072649, 0.04352542757987976, -0.009780683554708958, 0.034562088549137115, -0.032053593546152115, -0.05173061788082123, -0.044041719287633896, 0.01968034915626049, 0.007763845846056938, 0.05400868505239487, 0.04457078129053116], "cos_apical_negg": [0.007712585851550102, -0.03975548595190048, 0.025203227996826172, 0.03284638747572899, -0.02114245295524597, -0.02241712063550949, 0.01855619251728058, -0.011118985712528229, 0.09588839113712311, -0.023850148543715477, -0.046825066208839417, 0.003365863114595413, 0.0047249626368284225, 0.023461898788809776, -0.01765047386288643, 0.03156798332929611, -0.01771402545273304, 0.004477180540561676, 0.0037541198544204235, 0.035115797072649, 0.04352542757987976, -0.009780683554708958, 0.034562088549137115, -0.032053593546152115, -0.05173061788082123, -0.044041719287633896, 0.01968034915626049, 0.007763845846056938, 0.05400868505239487, 0.04457078129053116], "cos_Ac_negg": [0.007712585851550102, -0.03975548595190048, 0.025203227996826172, 0.03284638747572899, -0.02114245295524597, -0.02241712063550949, 0.01855619251728058, -0.011118985712528229, 0.09588839113712311, -0.023850148543715477, -0.046825066208839417, 0.003365863114595413, 0.0047249626368284225, 0.023461898788809776, -0.01765047386288643, 0.03156798332929611, -0.01771402545273304, 0.004477180540561676, 0.0037541198544204235, 0.035115797072649, 0.04352542757987976, -0.009780683554708958, 0.034562088549137115, -0.032053593546152115, -0.05173061788082123, -0.044041719287633896, 0.01968034915626049, 0.007763845846056938, 0.05400868505239487, 0.04457078129053116], "r_norm": [2.585965156555176, 2.487182140350342, 2.3580644130706787, 2.535940170288086, 2.390333652496338, 2.4134573936462402, 2.5035479068756104, 2.4554741382598877, 2.377804756164551, 2.3752760887145996, 2.4523282051086426, 2.3806276321411133, 2.4788317680358887, 2.501803398132324, 2.442944049835205, 2.4293007850646973, 2.429853916168213, 2.3894612789154053, 2.3824052810668945, 2.4184999465942383, 2.5621697902679443, 2.4488232135772705, 2.551830530166626, 2.4550294876098633, 2.407695770263672, 2.4057023525238037, 2.3515889644622803, 2.39849853515625, 2.419497489929199, 2.563612461090088], "ldr": {"dL_sdil": 0.0012927055358886719, "dL_bp": -0.35697102546691895, "ratio": -0.0036213178203967954}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3566, "test_loss": 1.8165755615234376}, {"step": 500, "epoch": 1, "train_loss": 1.762270450592041, "cos_r_negg": [0.6558809876441956, 0.07066728174686432, 0.018251970410346985, 0.08423072099685669, 0.014873884618282318, 0.09653906524181366, 0.06056039035320282, 0.014749763533473015, 0.11072409898042679, 0.10980172455310822, 0.06773938983678818, 0.06826259195804596, 0.055884283035993576, 0.017878936603665352, 0.058886896818876266, 0.09598827362060547, 0.03144843131303787, 0.03407925367355347, 0.05765664204955101, 0.008012301288545132, 0.04529644548892975, 0.05381857231259346, 0.11304923892021179, 0.08175028860569, 0.011697675101459026, 0.10300730168819427, 0.11308272182941437, 0.03464073687791824, 0.06281882524490356, 0.04331579804420471], "cos_innovation_negg": [0.6558809876441956, 0.07066728174686432, 0.018251970410346985, 0.08423072099685669, 0.014873884618282318, 0.09653906524181366, 0.06056039035320282, 0.014749763533473015, 0.11072409898042679, 0.10980172455310822, 0.06773938983678818, 0.06826259195804596, 0.055884283035993576, 0.017878936603665352, 0.058886896818876266, 0.09598827362060547, 0.03144843131303787, 0.03407925367355347, 0.05765664204955101, 0.008012301288545132, 0.04529644548892975, 0.05381857231259346, 0.11304923892021179, 0.08175028860569, 0.011697675101459026, 0.10300730168819427, 0.11308272182941437, 0.03464073687791824, 0.06281882524490356, 0.04331579804420471], "cos_apical_negg": [0.6558809876441956, 0.07066728174686432, 0.018251970410346985, 0.08423072099685669, 0.014873884618282318, 0.09653906524181366, 0.06056039035320282, 0.014749763533473015, 0.11072409898042679, 0.10980172455310822, 0.06773938983678818, 0.06826259195804596, 0.055884283035993576, 0.017878936603665352, 0.058886896818876266, 0.09598827362060547, 0.03144843131303787, 0.03407925367355347, 0.05765664204955101, 0.008012301288545132, 0.04529644548892975, 0.05381857231259346, 0.11304923892021179, 0.08175028860569, 0.011697675101459026, 0.10300730168819427, 0.11308272182941437, 0.03464073687791824, 0.06281882524490356, 0.04331579804420471], "cos_Ac_negg": [0.6558809876441956, 0.07066728174686432, 0.018251970410346985, 0.08423072099685669, 0.014873884618282318, 0.09653906524181366, 0.06056039035320282, 0.014749763533473015, 0.11072409898042679, 0.10980172455310822, 0.06773938983678818, 0.06826259195804596, 0.055884283035993576, 0.017878936603665352, 0.058886896818876266, 0.09598827362060547, 0.03144843131303787, 0.03407925367355347, 0.05765664204955101, 0.008012301288545132, 0.04529644548892975, 0.05381857231259346, 0.11304923892021179, 0.08175028860569, 0.011697675101459026, 0.10300730168819427, 0.11308272182941437, 0.03464073687791824, 0.06281882524490356, 0.04331579804420471], "r_norm": [1.8839044570922852, 1.7546790838241577, 1.665805697441101, 1.7955719232559204, 1.6785086393356323, 1.69894540309906, 1.7422223091125488, 1.721347451210022, 1.680301308631897, 1.6831727027893066, 1.7355031967163086, 1.6772198677062988, 1.762723445892334, 1.753968358039856, 1.7252929210662842, 1.7384850978851318, 1.716105580329895, 1.6800692081451416, 1.6614333391189575, 1.717505931854248, 1.781955599784851, 1.7441153526306152, 1.7911173105239868, 1.7119789123535156, 1.7116410732269287, 1.6895153522491455, 1.6457180976867676, 1.7111082077026367, 1.708539605140686, 1.8005788326263428]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3955, "test_loss": 1.745744482421875}, {"step": 1000, "epoch": 2, "train_loss": 1.677492380142212, "cos_r_negg": [0.7577265501022339, 0.1385941356420517, 0.11794496327638626, 0.1278281807899475, 0.06521080434322357, 0.23107996582984924, 0.14009007811546326, 0.09118485450744629, 0.1453491449356079, 0.15269191563129425, 0.12324953824281693, 0.16506412625312805, 0.13278870284557343, 0.11645059287548065, 0.14895015954971313, 0.15800440311431885, 0.06712011247873306, 0.08423195779323578, 0.1097240224480629, 0.07295048236846924, 0.08608043938875198, 0.1420779824256897, 0.2018738090991974, 0.1525341272354126, 0.13951443135738373, 0.16743025183677673, 0.19073817133903503, 0.11484451591968536, 0.12993669509887695, 0.14408499002456665], "cos_innovation_negg": [0.7577265501022339, 0.1385941356420517, 0.11794496327638626, 0.1278281807899475, 0.06521080434322357, 0.23107996582984924, 0.14009007811546326, 0.09118485450744629, 0.1453491449356079, 0.15269191563129425, 0.12324953824281693, 0.16506412625312805, 0.13278870284557343, 0.11645059287548065, 0.14895015954971313, 0.15800440311431885, 0.06712011247873306, 0.08423195779323578, 0.1097240224480629, 0.07295048236846924, 0.08608043938875198, 0.1420779824256897, 0.2018738090991974, 0.1525341272354126, 0.13951443135738373, 0.16743025183677673, 0.19073817133903503, 0.11484451591968536, 0.12993669509887695, 0.14408499002456665], "cos_apical_negg": [0.7577265501022339, 0.1385941356420517, 0.11794496327638626, 0.1278281807899475, 0.06521080434322357, 0.23107996582984924, 0.14009007811546326, 0.09118485450744629, 0.1453491449356079, 0.15269191563129425, 0.12324953824281693, 0.16506412625312805, 0.13278870284557343, 0.11645059287548065, 0.14895015954971313, 0.15800440311431885, 0.06712011247873306, 0.08423195779323578, 0.1097240224480629, 0.07295048236846924, 0.08608043938875198, 0.1420779824256897, 0.2018738090991974, 0.1525341272354126, 0.13951443135738373, 0.16743025183677673, 0.19073817133903503, 0.11484451591968536, 0.12993669509887695, 0.14408499002456665], "cos_Ac_negg": [0.7577265501022339, 0.1385941356420517, 0.11794496327638626, 0.1278281807899475, 0.06521080434322357, 0.23107996582984924, 0.14009007811546326, 0.09118485450744629, 0.1453491449356079, 0.15269191563129425, 0.12324953824281693, 0.16506412625312805, 0.13278870284557343, 0.11645059287548065, 0.14895015954971313, 0.15800440311431885, 0.06712011247873306, 0.08423195779323578, 0.1097240224480629, 0.07295048236846924, 0.08608043938875198, 0.1420779824256897, 0.2018738090991974, 0.1525341272354126, 0.13951443135738373, 0.16743025183677673, 0.19073817133903503, 0.11484451591968536, 0.12993669509887695, 0.14408499002456665], "r_norm": [1.6054446697235107, 1.4141788482666016, 1.3471059799194336, 1.4502768516540527, 1.3544790744781494, 1.3885712623596191, 1.4123278856277466, 1.3809483051300049, 1.3676725625991821, 1.3633958101272583, 1.4063048362731934, 1.3560385704040527, 1.417107105255127, 1.4133042097091675, 1.392833948135376, 1.4055331945419312, 1.3792095184326172, 1.3562562465667725, 1.340383768081665, 1.377598762512207, 1.4441884756088257, 1.4073444604873657, 1.453507900238037, 1.3987061977386475, 1.377488613128662, 1.3749264478683472, 1.3416244983673096, 1.3841962814331055, 1.3809940814971924, 1.4571963548660278]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4014, "test_loss": 1.721560546875}, {"step": 1500, "epoch": 3, "train_loss": 1.6506505012512207, "cos_r_negg": [0.7669909000396729, 0.2235923707485199, 0.201499342918396, 0.2277807593345642, 0.1550195962190628, 0.30092760920524597, 0.20241017639636993, 0.2018587291240692, 0.2682421803474426, 0.2446775734424591, 0.19383856654167175, 0.25659239292144775, 0.1819555163383484, 0.22689968347549438, 0.2452574074268341, 0.22027890384197235, 0.1669909656047821, 0.20393279194831848, 0.2563086748123169, 0.18761980533599854, 0.19737735390663147, 0.22478537261486053, 0.27947884798049927, 0.24358010292053223, 0.24352812767028809, 0.25636184215545654, 0.3240711987018585, 0.26102638244628906, 0.2404536008834839, 0.24738478660583496], "cos_innovation_negg": [0.7669909000396729, 0.2235923707485199, 0.201499342918396, 0.2277807593345642, 0.1550195962190628, 0.30092760920524597, 0.20241017639636993, 0.2018587291240692, 0.2682421803474426, 0.2446775734424591, 0.19383856654167175, 0.25659239292144775, 0.1819555163383484, 0.22689968347549438, 0.2452574074268341, 0.22027890384197235, 0.1669909656047821, 0.20393279194831848, 0.2563086748123169, 0.18761980533599854, 0.19737735390663147, 0.22478537261486053, 0.27947884798049927, 0.24358010292053223, 0.24352812767028809, 0.25636184215545654, 0.3240711987018585, 0.26102638244628906, 0.2404536008834839, 0.24738478660583496], "cos_apical_negg": [0.7669909000396729, 0.2235923707485199, 0.201499342918396, 0.2277807593345642, 0.1550195962190628, 0.30092760920524597, 0.20241017639636993, 0.2018587291240692, 0.2682421803474426, 0.2446775734424591, 0.19383856654167175, 0.25659239292144775, 0.1819555163383484, 0.22689968347549438, 0.2452574074268341, 0.22027890384197235, 0.1669909656047821, 0.20393279194831848, 0.2563086748123169, 0.18761980533599854, 0.19737735390663147, 0.22478537261486053, 0.27947884798049927, 0.24358010292053223, 0.24352812767028809, 0.25636184215545654, 0.3240711987018585, 0.26102638244628906, 0.2404536008834839, 0.24738478660583496], "cos_Ac_negg": [0.7669909000396729, 0.2235923707485199, 0.201499342918396, 0.2277807593345642, 0.1550195962190628, 0.30092760920524597, 0.20241017639636993, 0.2018587291240692, 0.2682421803474426, 0.2446775734424591, 0.19383856654167175, 0.25659239292144775, 0.1819555163383484, 0.22689968347549438, 0.2452574074268341, 0.22027890384197235, 0.1669909656047821, 0.20393279194831848, 0.2563086748123169, 0.18761980533599854, 0.19737735390663147, 0.22478537261486053, 0.27947884798049927, 0.24358010292053223, 0.24352812767028809, 0.25636184215545654, 0.3240711987018585, 0.26102638244628906, 0.2404536008834839, 0.24738478660583496], "r_norm": [1.3774014711380005, 1.1432597637176514, 1.083369255065918, 1.175041913986206, 1.0901926755905151, 1.1368950605392456, 1.1416852474212646, 1.111236572265625, 1.1139365434646606, 1.1099488735198975, 1.138288974761963, 1.1012654304504395, 1.144324541091919, 1.1400047540664673, 1.1340140104293823, 1.1459838151931763, 1.11038076877594, 1.1021370887756348, 1.093240737915039, 1.1028854846954346, 1.1586533784866333, 1.1386052370071411, 1.1901698112487793, 1.1362440586090088, 1.1151857376098633, 1.1161550283432007, 1.100908637046814, 1.1157726049423218, 1.120802640914917, 1.1745201349258423]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4076, "test_loss": 1.697406640625}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4146, "test_loss": 1.68923310546875}], "final": {"test_acc": 0.4146, "test_loss": 1.68923310546875, "wall_s": 48.79288172721863, "cos_r_negg": [0.7844280004501343, 0.2979603707790375, 0.30254095792770386, 0.31606239080429077, 0.285432904958725, 0.3901367783546448, 0.3063860535621643, 0.312701940536499, 0.3429080843925476, 0.36281368136405945, 0.2860463261604309, 0.3410678803920746, 0.30606693029403687, 0.3098071217536926, 0.3433891236782074, 0.337049663066864, 0.2844851016998291, 0.3203703761100769, 0.3284175992012024, 0.2743951082229614, 0.27600616216659546, 0.3081206679344177, 0.36312195658683777, 0.35184380412101746, 0.3211170434951782, 0.37896430492401123, 0.44468367099761963, 0.36475449800491333, 0.3426189124584198, 0.3603088855743408], "cos_innovation_negg": [0.7844280004501343, 0.2979603707790375, 0.30254095792770386, 0.31606239080429077, 0.285432904958725, 0.3901367783546448, 0.3063860535621643, 0.312701940536499, 0.3429080843925476, 0.36281368136405945, 0.2860463261604309, 0.3410678803920746, 0.30606693029403687, 0.3098071217536926, 0.3433891236782074, 0.337049663066864, 0.2844851016998291, 0.3203703761100769, 0.3284175992012024, 0.2743951082229614, 0.27600616216659546, 0.3081206679344177, 0.36312195658683777, 0.35184380412101746, 0.3211170434951782, 0.37896430492401123, 0.44468367099761963, 0.36475449800491333, 0.3426189124584198, 0.3603088855743408], "cos_apical_negg": [0.7844280004501343, 0.2979603707790375, 0.30254095792770386, 0.31606239080429077, 0.285432904958725, 0.3901367783546448, 0.3063860535621643, 0.312701940536499, 0.3429080843925476, 0.36281368136405945, 0.2860463261604309, 0.3410678803920746, 0.30606693029403687, 0.3098071217536926, 0.3433891236782074, 0.337049663066864, 0.2844851016998291, 0.3203703761100769, 0.3284175992012024, 0.2743951082229614, 0.27600616216659546, 0.3081206679344177, 0.36312195658683777, 0.35184380412101746, 0.3211170434951782, 0.37896430492401123, 0.44468367099761963, 0.36475449800491333, 0.3426189124584198, 0.3603088855743408], "cos_Ac_negg": [0.7844280004501343, 0.2979603707790375, 0.30254095792770386, 0.31606239080429077, 0.285432904958725, 0.3901367783546448, 0.3063860535621643, 0.312701940536499, 0.3429080843925476, 0.36281368136405945, 0.2860463261604309, 0.3410678803920746, 0.30606693029403687, 0.3098071217536926, 0.3433891236782074, 0.337049663066864, 0.2844851016998291, 0.3203703761100769, 0.3284175992012024, 0.2743951082229614, 0.27600616216659546, 0.3081206679344177, 0.36312195658683777, 0.35184380412101746, 0.3211170434951782, 0.37896430492401123, 0.44468367099761963, 0.36475449800491333, 0.3426189124584198, 0.3603088855743408]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d30_s1.json b/results/scale_v3_cifar10_sdil_w64_d30_s1.json new file mode 100644 index 0000000..29e8721 --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d30_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d30_s1", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.638411045074463, "cos_r_negg": [0.08691532909870148, -0.055161379277706146, 0.00924270786345005, -0.007116837427020073, 0.03401412442326546, -0.041550397872924805, -0.03714495897293091, 0.06715098768472672, 0.024935483932495117, 0.004849172197282314, 0.03067401982843876, -6.456300616264343e-05, -0.050803475081920624, -0.08004932105541229, 0.08344809710979462, 0.008492517285048962, 0.04852515459060669, -0.08461719751358032, 0.027081595733761787, 0.025112375617027283, -0.048559486865997314, -0.0018642422510311007, -0.005842576734721661, -0.04324054718017578, 0.012212688103318214, 0.07562638819217682, -0.01945127546787262, -0.012839537113904953, -0.08922368288040161, 0.03311462700366974], "cos_innovation_negg": [0.08691532909870148, -0.055161379277706146, 0.00924270786345005, -0.007116837427020073, 0.03401412442326546, -0.041550397872924805, -0.03714495897293091, 0.06715098768472672, 0.024935483932495117, 0.004849172197282314, 0.03067401982843876, -6.456300616264343e-05, -0.050803475081920624, -0.08004932105541229, 0.08344809710979462, 0.008492517285048962, 0.04852515459060669, -0.08461719751358032, 0.027081595733761787, 0.025112375617027283, -0.048559486865997314, -0.0018642422510311007, -0.005842576734721661, -0.04324054718017578, 0.012212688103318214, 0.07562638819217682, -0.01945127546787262, -0.012839537113904953, -0.08922368288040161, 0.03311462700366974], "cos_apical_negg": [0.08691532909870148, -0.055161379277706146, 0.00924270786345005, -0.007116837427020073, 0.03401412442326546, -0.041550397872924805, -0.03714495897293091, 0.06715098768472672, 0.024935483932495117, 0.004849172197282314, 0.03067401982843876, -6.456300616264343e-05, -0.050803475081920624, -0.08004932105541229, 0.08344809710979462, 0.008492517285048962, 0.04852515459060669, -0.08461719751358032, 0.027081595733761787, 0.025112375617027283, -0.048559486865997314, -0.0018642422510311007, -0.005842576734721661, -0.04324054718017578, 0.012212688103318214, 0.07562638819217682, -0.01945127546787262, -0.012839537113904953, -0.08922368288040161, 0.03311462700366974], "cos_Ac_negg": [0.08691532909870148, -0.055161379277706146, 0.00924270786345005, -0.007116837427020073, 0.03401412442326546, -0.041550397872924805, -0.03714495897293091, 0.06715098768472672, 0.024935483932495117, 0.004849172197282314, 0.03067401982843876, -6.456300616264343e-05, -0.050803475081920624, -0.08004932105541229, 0.08344809710979462, 0.008492517285048962, 0.04852515459060669, -0.08461719751358032, 0.027081595733761787, 0.025112375617027283, -0.048559486865997314, -0.0018642422510311007, -0.005842576734721661, -0.04324054718017578, 0.012212688103318214, 0.07562638819217682, -0.01945127546787262, -0.012839537113904953, -0.08922368288040161, 0.03311462700366974], "r_norm": [2.346062660217285, 2.3706045150756836, 2.4505882263183594, 2.378006935119629, 2.587252616882324, 2.528420925140381, 2.5919618606567383, 2.430840492248535, 2.425443649291992, 2.536597490310669, 2.514082431793213, 2.53230357170105, 2.377021312713623, 2.465062141418457, 2.5614004135131836, 2.368165969848633, 2.4030864238739014, 2.60673189163208, 2.4764347076416016, 2.5003862380981445, 2.380082130432129, 2.4691433906555176, 2.610625743865967, 2.520176410675049, 2.391207695007324, 2.3642208576202393, 2.547528028488159, 2.537539005279541, 2.367097854614258, 2.407374620437622], "ldr": {"dL_sdil": -0.1189737319946289, "dL_bp": -0.4420166015625, "ratio": 0.2691612296326982}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3702, "test_loss": 1.79817431640625}, {"step": 500, "epoch": 1, "train_loss": 1.714444637298584, "cos_r_negg": [0.6082881093025208, 0.15209367871284485, -0.0002454817295074463, 0.11812738329172134, -0.009981494396924973, 0.04905594140291214, 0.08803354203701019, 0.05667778104543686, 0.13373330235481262, 0.0003694235347211361, 0.04648260399699211, 0.04151448607444763, 0.09155389666557312, 0.08384542167186737, 0.09610782563686371, 0.031241174787282944, 0.12847955524921417, 0.07399386167526245, 0.0742131844162941, 0.05936291441321373, 0.1588454246520996, 0.09620848298072815, 0.09967698156833649, 0.0003706207498908043, 0.06009450554847717, 0.0923810601234436, -0.03997114300727844, -0.018029596656560898, 0.04846490919589996, 0.06254607439041138], "cos_innovation_negg": [0.6082881093025208, 0.15209367871284485, -0.0002454817295074463, 0.11812738329172134, -0.009981494396924973, 0.04905594140291214, 0.08803354203701019, 0.05667778104543686, 0.13373330235481262, 0.0003694235347211361, 0.04648260399699211, 0.04151448607444763, 0.09155389666557312, 0.08384542167186737, 0.09610782563686371, 0.031241174787282944, 0.12847955524921417, 0.07399386167526245, 0.0742131844162941, 0.05936291441321373, 0.1588454246520996, 0.09620848298072815, 0.09967698156833649, 0.0003706207498908043, 0.06009450554847717, 0.0923810601234436, -0.03997114300727844, -0.018029596656560898, 0.04846490919589996, 0.06254607439041138], "cos_apical_negg": [0.6082881093025208, 0.15209367871284485, -0.0002454817295074463, 0.11812738329172134, -0.009981494396924973, 0.04905594140291214, 0.08803354203701019, 0.05667778104543686, 0.13373330235481262, 0.0003694235347211361, 0.04648260399699211, 0.04151448607444763, 0.09155389666557312, 0.08384542167186737, 0.09610782563686371, 0.031241174787282944, 0.12847955524921417, 0.07399386167526245, 0.0742131844162941, 0.05936291441321373, 0.1588454246520996, 0.09620848298072815, 0.09967698156833649, 0.0003706207498908043, 0.06009450554847717, 0.0923810601234436, -0.03997114300727844, -0.018029596656560898, 0.04846490919589996, 0.06254607439041138], "cos_Ac_negg": [0.6082881093025208, 0.15209367871284485, -0.0002454817295074463, 0.11812738329172134, -0.009981494396924973, 0.04905594140291214, 0.08803354203701019, 0.05667778104543686, 0.13373330235481262, 0.0003694235347211361, 0.04648260399699211, 0.04151448607444763, 0.09155389666557312, 0.08384542167186737, 0.09610782563686371, 0.031241174787282944, 0.12847955524921417, 0.07399386167526245, 0.0742131844162941, 0.05936291441321373, 0.1588454246520996, 0.09620848298072815, 0.09967698156833649, 0.0003706207498908043, 0.06009450554847717, 0.0923810601234436, -0.03997114300727844, -0.018029596656560898, 0.04846490919589996, 0.06254607439041138], "r_norm": [1.6733005046844482, 1.620328426361084, 1.669677495956421, 1.6463899612426758, 1.7458769083023071, 1.7417759895324707, 1.770848035812378, 1.663561463356018, 1.6639585494995117, 1.7375514507293701, 1.7149546146392822, 1.7191296815872192, 1.6307027339935303, 1.6791609525680542, 1.7783012390136719, 1.631772756576538, 1.6507532596588135, 1.760416030883789, 1.6983038187026978, 1.6939589977264404, 1.643247127532959, 1.6996867656707764, 1.7872592210769653, 1.725723385810852, 1.6372641324996948, 1.6350620985031128, 1.744750738143921, 1.7218668460845947, 1.6212950944900513, 1.6589484214782715]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3917, "test_loss": 1.7512622192382812}, {"step": 1000, "epoch": 2, "train_loss": 1.708784580230713, "cos_r_negg": [0.7159529328346252, 0.20893211662769318, 0.08470971882343292, 0.1606137901544571, 0.06614220142364502, 0.08110655844211578, 0.1417987048625946, 0.10778505355119705, 0.22893023490905762, 0.06980562210083008, 0.1184537410736084, 0.1289384365081787, 0.1910615861415863, 0.12169494479894638, 0.14882588386535645, 0.1404440850019455, 0.22272557020187378, 0.19879871606826782, 0.15975920855998993, 0.15750718116760254, 0.20302867889404297, 0.16645056009292603, 0.14143161475658417, 0.1247381791472435, 0.14499932527542114, 0.1294010579586029, 0.04271066188812256, 0.036660969257354736, 0.1023494079709053, 0.09002550691366196], "cos_innovation_negg": [0.7159529328346252, 0.20893211662769318, 0.08470971882343292, 0.1606137901544571, 0.06614220142364502, 0.08110655844211578, 0.1417987048625946, 0.10778505355119705, 0.22893023490905762, 0.06980562210083008, 0.1184537410736084, 0.1289384365081787, 0.1910615861415863, 0.12169494479894638, 0.14882588386535645, 0.1404440850019455, 0.22272557020187378, 0.19879871606826782, 0.15975920855998993, 0.15750718116760254, 0.20302867889404297, 0.16645056009292603, 0.14143161475658417, 0.1247381791472435, 0.14499932527542114, 0.1294010579586029, 0.04271066188812256, 0.036660969257354736, 0.1023494079709053, 0.09002550691366196], "cos_apical_negg": [0.7159529328346252, 0.20893211662769318, 0.08470971882343292, 0.1606137901544571, 0.06614220142364502, 0.08110655844211578, 0.1417987048625946, 0.10778505355119705, 0.22893023490905762, 0.06980562210083008, 0.1184537410736084, 0.1289384365081787, 0.1910615861415863, 0.12169494479894638, 0.14882588386535645, 0.1404440850019455, 0.22272557020187378, 0.19879871606826782, 0.15975920855998993, 0.15750718116760254, 0.20302867889404297, 0.16645056009292603, 0.14143161475658417, 0.1247381791472435, 0.14499932527542114, 0.1294010579586029, 0.04271066188812256, 0.036660969257354736, 0.1023494079709053, 0.09002550691366196], "cos_Ac_negg": [0.7159529328346252, 0.20893211662769318, 0.08470971882343292, 0.1606137901544571, 0.06614220142364502, 0.08110655844211578, 0.1417987048625946, 0.10778505355119705, 0.22893023490905762, 0.06980562210083008, 0.1184537410736084, 0.1289384365081787, 0.1910615861415863, 0.12169494479894638, 0.14882588386535645, 0.1404440850019455, 0.22272557020187378, 0.19879871606826782, 0.15975920855998993, 0.15750718116760254, 0.20302867889404297, 0.16645056009292603, 0.14143161475658417, 0.1247381791472435, 0.14499932527542114, 0.1294010579586029, 0.04271066188812256, 0.036660969257354736, 0.1023494079709053, 0.09002550691366196], "r_norm": [1.4629340171813965, 1.3450825214385986, 1.3747920989990234, 1.3620078563690186, 1.4349610805511475, 1.4289660453796387, 1.467505931854248, 1.3675472736358643, 1.3870267868041992, 1.4248181581497192, 1.4206510782241821, 1.4238088130950928, 1.3563538789749146, 1.3951032161712646, 1.463655710220337, 1.355128526687622, 1.3737595081329346, 1.45356285572052, 1.402052402496338, 1.4049384593963623, 1.365801215171814, 1.4092814922332764, 1.4852542877197266, 1.4229356050491333, 1.356194257736206, 1.358608365058899, 1.4261736869812012, 1.413891315460205, 1.3364571332931519, 1.3784403800964355]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3903, "test_loss": 1.7335889404296876}, {"step": 1500, "epoch": 3, "train_loss": 1.6689519882202148, "cos_r_negg": [0.7754727602005005, 0.26361507177352905, 0.15877532958984375, 0.27105259895324707, 0.15586401522159576, 0.18557611107826233, 0.2184038758277893, 0.2123415768146515, 0.31685778498649597, 0.1662488877773285, 0.20819950103759766, 0.21610575914382935, 0.22973763942718506, 0.23454146087169647, 0.2740199565887451, 0.26250070333480835, 0.3067709803581238, 0.26112648844718933, 0.2618536353111267, 0.24590378999710083, 0.2916918992996216, 0.2280527949333191, 0.23624417185783386, 0.19200217723846436, 0.21173489093780518, 0.23020626604557037, 0.13038602471351624, 0.14458893239498138, 0.17873579263687134, 0.23093004524707794], "cos_innovation_negg": [0.7754727602005005, 0.26361507177352905, 0.15877532958984375, 0.27105259895324707, 0.15586401522159576, 0.18557611107826233, 0.2184038758277893, 0.2123415768146515, 0.31685778498649597, 0.1662488877773285, 0.20819950103759766, 0.21610575914382935, 0.22973763942718506, 0.23454146087169647, 0.2740199565887451, 0.26250070333480835, 0.3067709803581238, 0.26112648844718933, 0.2618536353111267, 0.24590378999710083, 0.2916918992996216, 0.2280527949333191, 0.23624417185783386, 0.19200217723846436, 0.21173489093780518, 0.23020626604557037, 0.13038602471351624, 0.14458893239498138, 0.17873579263687134, 0.23093004524707794], "cos_apical_negg": [0.7754727602005005, 0.26361507177352905, 0.15877532958984375, 0.27105259895324707, 0.15586401522159576, 0.18557611107826233, 0.2184038758277893, 0.2123415768146515, 0.31685778498649597, 0.1662488877773285, 0.20819950103759766, 0.21610575914382935, 0.22973763942718506, 0.23454146087169647, 0.2740199565887451, 0.26250070333480835, 0.3067709803581238, 0.26112648844718933, 0.2618536353111267, 0.24590378999710083, 0.2916918992996216, 0.2280527949333191, 0.23624417185783386, 0.19200217723846436, 0.21173489093780518, 0.23020626604557037, 0.13038602471351624, 0.14458893239498138, 0.17873579263687134, 0.23093004524707794], "cos_Ac_negg": [0.7754727602005005, 0.26361507177352905, 0.15877532958984375, 0.27105259895324707, 0.15586401522159576, 0.18557611107826233, 0.2184038758277893, 0.2123415768146515, 0.31685778498649597, 0.1662488877773285, 0.20819950103759766, 0.21610575914382935, 0.22973763942718506, 0.23454146087169647, 0.2740199565887451, 0.26250070333480835, 0.3067709803581238, 0.26112648844718933, 0.2618536353111267, 0.24590378999710083, 0.2916918992996216, 0.2280527949333191, 0.23624417185783386, 0.19200217723846436, 0.21173489093780518, 0.23020626604557037, 0.13038602471351624, 0.14458893239498138, 0.17873579263687134, 0.23093004524707794], "r_norm": [1.2811216115951538, 1.1123378276824951, 1.128166913986206, 1.1171106100082397, 1.1847177743911743, 1.170053482055664, 1.1962863206863403, 1.1259255409240723, 1.1498655080795288, 1.1605644226074219, 1.1605470180511475, 1.1709702014923096, 1.1173055171966553, 1.1460299491882324, 1.203818440437317, 1.1115821599960327, 1.1357207298278809, 1.2022299766540527, 1.153313159942627, 1.1639389991760254, 1.1262948513031006, 1.1589338779449463, 1.2143644094467163, 1.1657297611236572, 1.1105802059173584, 1.1109596490859985, 1.1557074785232544, 1.1506805419921875, 1.0889925956726074, 1.1316322088241577]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4031, "test_loss": 1.7158672241210937}, {"epoch_end": 4, "step": 1955, "test_acc": 0.407, "test_loss": 1.7061551635742187}], "final": {"test_acc": 0.407, "test_loss": 1.7061551635742187, "wall_s": 48.24258375167847, "cos_r_negg": [0.7782910466194153, 0.36765754222869873, 0.2886629104614258, 0.32784369587898254, 0.24559742212295532, 0.2890312671661377, 0.3117924928665161, 0.32073962688446045, 0.36459940671920776, 0.2523760199546814, 0.3397488296031952, 0.31432002782821655, 0.3705013692378998, 0.35510480403900146, 0.35948097705841064, 0.36500537395477295, 0.39019840955734253, 0.33081576228141785, 0.3589869737625122, 0.3514150083065033, 0.3829733729362488, 0.32762372493743896, 0.3427068293094635, 0.3179625868797302, 0.339911550283432, 0.34203800559043884, 0.23064303398132324, 0.27240604162216187, 0.29243555665016174, 0.3428814113140106], "cos_innovation_negg": [0.7782910466194153, 0.36765754222869873, 0.2886629104614258, 0.32784369587898254, 0.24559742212295532, 0.2890312671661377, 0.3117924928665161, 0.32073962688446045, 0.36459940671920776, 0.2523760199546814, 0.3397488296031952, 0.31432002782821655, 0.3705013692378998, 0.35510480403900146, 0.35948097705841064, 0.36500537395477295, 0.39019840955734253, 0.33081576228141785, 0.3589869737625122, 0.3514150083065033, 0.3829733729362488, 0.32762372493743896, 0.3427068293094635, 0.3179625868797302, 0.339911550283432, 0.34203800559043884, 0.23064303398132324, 0.27240604162216187, 0.29243555665016174, 0.3428814113140106], "cos_apical_negg": [0.7782910466194153, 0.36765754222869873, 0.2886629104614258, 0.32784369587898254, 0.24559742212295532, 0.2890312671661377, 0.3117924928665161, 0.32073962688446045, 0.36459940671920776, 0.2523760199546814, 0.3397488296031952, 0.31432002782821655, 0.3705013692378998, 0.35510480403900146, 0.35948097705841064, 0.36500537395477295, 0.39019840955734253, 0.33081576228141785, 0.3589869737625122, 0.3514150083065033, 0.3829733729362488, 0.32762372493743896, 0.3427068293094635, 0.3179625868797302, 0.339911550283432, 0.34203800559043884, 0.23064303398132324, 0.27240604162216187, 0.29243555665016174, 0.3428814113140106], "cos_Ac_negg": [0.7782910466194153, 0.36765754222869873, 0.2886629104614258, 0.32784369587898254, 0.24559742212295532, 0.2890312671661377, 0.3117924928665161, 0.32073962688446045, 0.36459940671920776, 0.2523760199546814, 0.3397488296031952, 0.31432002782821655, 0.3705013692378998, 0.35510480403900146, 0.35948097705841064, 0.36500537395477295, 0.39019840955734253, 0.33081576228141785, 0.3589869737625122, 0.3514150083065033, 0.3829733729362488, 0.32762372493743896, 0.3427068293094635, 0.3179625868797302, 0.339911550283432, 0.34203800559043884, 0.23064303398132324, 0.27240604162216187, 0.29243555665016174, 0.3428814113140106]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d30_s2.json b/results/scale_v3_cifar10_sdil_w64_d30_s2.json new file mode 100644 index 0000000..881e84f --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d30_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d30_s2", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.7673251628875732, "cos_r_negg": [-0.0018449039198458195, 0.043090566992759705, -0.030655743554234505, 0.044208724051713943, -0.03918180614709854, -0.007073882035911083, -0.07245612144470215, 0.007156517822295427, 0.04394353926181793, 0.01088100578635931, 0.018909847363829613, -0.04214581102132797, -0.03510477393865585, 0.03821041062474251, -0.04766027256846428, 0.0332387313246727, -0.01074511744081974, -0.07070790231227875, -0.0869806557893753, -0.010209904983639717, 0.03801646828651428, 0.07088203728199005, -0.04375465586781502, 0.06568114459514618, 0.006486187689006329, 0.03687378764152527, -0.01828612945973873, -0.015622059814631939, -0.02161308005452156, -0.012575682252645493], "cos_innovation_negg": [-0.0018449039198458195, 0.043090566992759705, -0.030655743554234505, 0.044208724051713943, -0.03918180614709854, -0.007073882035911083, -0.07245612144470215, 0.007156517822295427, 0.04394353926181793, 0.01088100578635931, 0.018909847363829613, -0.04214581102132797, -0.03510477393865585, 0.03821041062474251, -0.04766027256846428, 0.0332387313246727, -0.01074511744081974, -0.07070790231227875, -0.0869806557893753, -0.010209904983639717, 0.03801646828651428, 0.07088203728199005, -0.04375465586781502, 0.06568114459514618, 0.006486187689006329, 0.03687378764152527, -0.01828612945973873, -0.015622059814631939, -0.02161308005452156, -0.012575682252645493], "cos_apical_negg": [-0.0018449039198458195, 0.043090566992759705, -0.030655743554234505, 0.044208724051713943, -0.03918180614709854, -0.007073882035911083, -0.07245612144470215, 0.007156517822295427, 0.04394353926181793, 0.01088100578635931, 0.018909847363829613, -0.04214581102132797, -0.03510477393865585, 0.03821041062474251, -0.04766027256846428, 0.0332387313246727, -0.01074511744081974, -0.07070790231227875, -0.0869806557893753, -0.010209904983639717, 0.03801646828651428, 0.07088203728199005, -0.04375465586781502, 0.06568114459514618, 0.006486187689006329, 0.03687378764152527, -0.01828612945973873, -0.015622059814631939, -0.02161308005452156, -0.012575682252645493], "cos_Ac_negg": [-0.0018449039198458195, 0.043090566992759705, -0.030655743554234505, 0.044208724051713943, -0.03918180614709854, -0.007073882035911083, -0.07245612144470215, 0.007156517822295427, 0.04394353926181793, 0.01088100578635931, 0.018909847363829613, -0.04214581102132797, -0.03510477393865585, 0.03821041062474251, -0.04766027256846428, 0.0332387313246727, -0.01074511744081974, -0.07070790231227875, -0.0869806557893753, -0.010209904983639717, 0.03801646828651428, 0.07088203728199005, -0.04375465586781502, 0.06568114459514618, 0.006486187689006329, 0.03687378764152527, -0.01828612945973873, -0.015622059814631939, -0.02161308005452156, -0.012575682252645493], "r_norm": [2.430849552154541, 2.4477221965789795, 2.3842697143554688, 2.419781446456909, 2.3784422874450684, 2.478245735168457, 2.471724033355713, 2.577846050262451, 2.604187488555908, 2.4785141944885254, 2.682030439376831, 2.4370269775390625, 2.399013042449951, 2.5771358013153076, 2.4997215270996094, 2.376765727996826, 2.362417697906494, 2.4595911502838135, 2.429932117462158, 2.4942843914031982, 2.528311252593994, 2.4075803756713867, 2.4573755264282227, 2.503483772277832, 2.437056064605713, 2.601693630218506, 2.44462513923645, 2.487914800643921, 2.4807746410369873, 2.446587562561035], "ldr": {"dL_sdil": -0.06371641159057617, "dL_bp": -0.4305086135864258, "ratio": 0.14800264055061682}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3622, "test_loss": 1.8087956298828125}, {"step": 500, "epoch": 1, "train_loss": 1.699876070022583, "cos_r_negg": [0.6231592893600464, 0.03234781697392464, -0.013537107035517693, 0.06719902157783508, 0.09888224303722382, 0.11364051699638367, 0.02124936133623123, 0.04318549484014511, -0.04911576956510544, 0.10597953200340271, 0.05600105971097946, -0.04411859065294266, 0.05726641044020653, 0.055515024811029434, 0.055337220430374146, 0.06593811511993408, 0.10628067702054977, -0.003538860008120537, 0.05356182903051376, -0.03395355865359306, 0.09674793481826782, -0.02772381901741028, 0.15566550195217133, 0.08221966028213501, 0.08790072798728943, 0.04558678716421127, 0.08405075967311859, 0.019825352355837822, 0.040003061294555664, -0.02139369770884514], "cos_innovation_negg": [0.6231592893600464, 0.03234781697392464, -0.013537107035517693, 0.06719902157783508, 0.09888224303722382, 0.11364051699638367, 0.02124936133623123, 0.04318549484014511, -0.04911576956510544, 0.10597953200340271, 0.05600105971097946, -0.04411859065294266, 0.05726641044020653, 0.055515024811029434, 0.055337220430374146, 0.06593811511993408, 0.10628067702054977, -0.003538860008120537, 0.05356182903051376, -0.03395355865359306, 0.09674793481826782, -0.02772381901741028, 0.15566550195217133, 0.08221966028213501, 0.08790072798728943, 0.04558678716421127, 0.08405075967311859, 0.019825352355837822, 0.040003061294555664, -0.02139369770884514], "cos_apical_negg": [0.6231592893600464, 0.03234781697392464, -0.013537107035517693, 0.06719902157783508, 0.09888224303722382, 0.11364051699638367, 0.02124936133623123, 0.04318549484014511, -0.04911576956510544, 0.10597953200340271, 0.05600105971097946, -0.04411859065294266, 0.05726641044020653, 0.055515024811029434, 0.055337220430374146, 0.06593811511993408, 0.10628067702054977, -0.003538860008120537, 0.05356182903051376, -0.03395355865359306, 0.09674793481826782, -0.02772381901741028, 0.15566550195217133, 0.08221966028213501, 0.08790072798728943, 0.04558678716421127, 0.08405075967311859, 0.019825352355837822, 0.040003061294555664, -0.02139369770884514], "cos_Ac_negg": [0.6231592893600464, 0.03234781697392464, -0.013537107035517693, 0.06719902157783508, 0.09888224303722382, 0.11364051699638367, 0.02124936133623123, 0.04318549484014511, -0.04911576956510544, 0.10597953200340271, 0.05600105971097946, -0.04411859065294266, 0.05726641044020653, 0.055515024811029434, 0.055337220430374146, 0.06593811511993408, 0.10628067702054977, -0.003538860008120537, 0.05356182903051376, -0.03395355865359306, 0.09674793481826782, -0.02772381901741028, 0.15566550195217133, 0.08221966028213501, 0.08790072798728943, 0.04558678716421127, 0.08405075967311859, 0.019825352355837822, 0.040003061294555664, -0.02139369770884514], "r_norm": [1.7524669170379639, 1.7085061073303223, 1.6562848091125488, 1.6905086040496826, 1.6490436792373657, 1.741104006767273, 1.7217624187469482, 1.7878477573394775, 1.797762155532837, 1.733629822731018, 1.8833401203155518, 1.676222562789917, 1.671111822128296, 1.7828971147537231, 1.7308093309402466, 1.653311848640442, 1.6448994874954224, 1.7019076347351074, 1.6843132972717285, 1.742912769317627, 1.771352767944336, 1.6765071153640747, 1.7338296175003052, 1.7312037944793701, 1.707879900932312, 1.826425313949585, 1.7140750885009766, 1.7241415977478027, 1.722238540649414, 1.6934146881103516]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3853, "test_loss": 1.7610109985351563}, {"step": 1000, "epoch": 2, "train_loss": 1.7009296417236328, "cos_r_negg": [0.7149951457977295, 0.13175418972969055, 0.06972996890544891, 0.12254331260919571, 0.1407756507396698, 0.17054907977581024, 0.08395099639892578, 0.14854440093040466, 0.05981184542179108, 0.20346319675445557, 0.0882091075181961, 0.08340004086494446, 0.1413470357656479, 0.15138830244541168, 0.13958576321601868, 0.10893744230270386, 0.16541443765163422, 0.08835472166538239, 0.1319752037525177, 0.06233470141887665, 0.15439075231552124, 0.0538264736533165, 0.20302805304527283, 0.11336147785186768, 0.1683836579322815, 0.1357850581407547, 0.18407396972179413, 0.062275052070617676, 0.16772544384002686, 0.04878085106611252], "cos_innovation_negg": [0.7149951457977295, 0.13175418972969055, 0.06972996890544891, 0.12254331260919571, 0.1407756507396698, 0.17054907977581024, 0.08395099639892578, 0.14854440093040466, 0.05981184542179108, 0.20346319675445557, 0.0882091075181961, 0.08340004086494446, 0.1413470357656479, 0.15138830244541168, 0.13958576321601868, 0.10893744230270386, 0.16541443765163422, 0.08835472166538239, 0.1319752037525177, 0.06233470141887665, 0.15439075231552124, 0.0538264736533165, 0.20302805304527283, 0.11336147785186768, 0.1683836579322815, 0.1357850581407547, 0.18407396972179413, 0.062275052070617676, 0.16772544384002686, 0.04878085106611252], "cos_apical_negg": [0.7149951457977295, 0.13175418972969055, 0.06972996890544891, 0.12254331260919571, 0.1407756507396698, 0.17054907977581024, 0.08395099639892578, 0.14854440093040466, 0.05981184542179108, 0.20346319675445557, 0.0882091075181961, 0.08340004086494446, 0.1413470357656479, 0.15138830244541168, 0.13958576321601868, 0.10893744230270386, 0.16541443765163422, 0.08835472166538239, 0.1319752037525177, 0.06233470141887665, 0.15439075231552124, 0.0538264736533165, 0.20302805304527283, 0.11336147785186768, 0.1683836579322815, 0.1357850581407547, 0.18407396972179413, 0.062275052070617676, 0.16772544384002686, 0.04878085106611252], "cos_Ac_negg": [0.7149951457977295, 0.13175418972969055, 0.06972996890544891, 0.12254331260919571, 0.1407756507396698, 0.17054907977581024, 0.08395099639892578, 0.14854440093040466, 0.05981184542179108, 0.20346319675445557, 0.0882091075181961, 0.08340004086494446, 0.1413470357656479, 0.15138830244541168, 0.13958576321601868, 0.10893744230270386, 0.16541443765163422, 0.08835472166538239, 0.1319752037525177, 0.06233470141887665, 0.15439075231552124, 0.0538264736533165, 0.20302805304527283, 0.11336147785186768, 0.1683836579322815, 0.1357850581407547, 0.18407396972179413, 0.062275052070617676, 0.16772544384002686, 0.04878085106611252], "r_norm": [1.4920341968536377, 1.3695182800292969, 1.32114577293396, 1.360193133354187, 1.330500841140747, 1.4078905582427979, 1.3869109153747559, 1.436488151550293, 1.4383617639541626, 1.3985425233840942, 1.5073630809783936, 1.3447448015213013, 1.3489863872528076, 1.439378261566162, 1.3960191011428833, 1.3299473524093628, 1.3348543643951416, 1.3642886877059937, 1.3563181161880493, 1.3798013925552368, 1.4326658248901367, 1.3444747924804688, 1.4027411937713623, 1.3910826444625854, 1.3769726753234863, 1.4678550958633423, 1.3837158679962158, 1.3808398246765137, 1.3865352869033813, 1.3585736751556396]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3978, "test_loss": 1.7260768310546875}, {"step": 1500, "epoch": 3, "train_loss": 1.663680076599121, "cos_r_negg": [0.7351300120353699, 0.22837603092193604, 0.1699596494436264, 0.22487905621528625, 0.2139565795660019, 0.28815263509750366, 0.19904375076293945, 0.2326345443725586, 0.16562718152999878, 0.2702944278717041, 0.1571425497531891, 0.18601398169994354, 0.24768587946891785, 0.20362553000450134, 0.22826528549194336, 0.20807074010372162, 0.2876613736152649, 0.20435801148414612, 0.2255106270313263, 0.13609914481639862, 0.27329158782958984, 0.15789484977722168, 0.29697054624557495, 0.20620682835578918, 0.2591324746608734, 0.23347464203834534, 0.27815625071525574, 0.1651098132133484, 0.2608111500740051, 0.17800763249397278], "cos_innovation_negg": [0.7351300120353699, 0.22837603092193604, 0.1699596494436264, 0.22487905621528625, 0.2139565795660019, 0.28815263509750366, 0.19904375076293945, 0.2326345443725586, 0.16562718152999878, 0.2702944278717041, 0.1571425497531891, 0.18601398169994354, 0.24768587946891785, 0.20362553000450134, 0.22826528549194336, 0.20807074010372162, 0.2876613736152649, 0.20435801148414612, 0.2255106270313263, 0.13609914481639862, 0.27329158782958984, 0.15789484977722168, 0.29697054624557495, 0.20620682835578918, 0.2591324746608734, 0.23347464203834534, 0.27815625071525574, 0.1651098132133484, 0.2608111500740051, 0.17800763249397278], "cos_apical_negg": [0.7351300120353699, 0.22837603092193604, 0.1699596494436264, 0.22487905621528625, 0.2139565795660019, 0.28815263509750366, 0.19904375076293945, 0.2326345443725586, 0.16562718152999878, 0.2702944278717041, 0.1571425497531891, 0.18601398169994354, 0.24768587946891785, 0.20362553000450134, 0.22826528549194336, 0.20807074010372162, 0.2876613736152649, 0.20435801148414612, 0.2255106270313263, 0.13609914481639862, 0.27329158782958984, 0.15789484977722168, 0.29697054624557495, 0.20620682835578918, 0.2591324746608734, 0.23347464203834534, 0.27815625071525574, 0.1651098132133484, 0.2608111500740051, 0.17800763249397278], "cos_Ac_negg": [0.7351300120353699, 0.22837603092193604, 0.1699596494436264, 0.22487905621528625, 0.2139565795660019, 0.28815263509750366, 0.19904375076293945, 0.2326345443725586, 0.16562718152999878, 0.2702944278717041, 0.1571425497531891, 0.18601398169994354, 0.24768587946891785, 0.20362553000450134, 0.22826528549194336, 0.20807074010372162, 0.2876613736152649, 0.20435801148414612, 0.2255106270313263, 0.13609914481639862, 0.27329158782958984, 0.15789484977722168, 0.29697054624557495, 0.20620682835578918, 0.2591324746608734, 0.23347464203834534, 0.27815625071525574, 0.1651098132133484, 0.2608111500740051, 0.17800763249397278], "r_norm": [1.300966501235962, 1.124415636062622, 1.0868703126907349, 1.1216444969177246, 1.0935983657836914, 1.163970708847046, 1.1378521919250488, 1.182159423828125, 1.1744301319122314, 1.1557888984680176, 1.232257604598999, 1.0937950611114502, 1.1133556365966797, 1.1867964267730713, 1.143052101135254, 1.097116470336914, 1.1072778701782227, 1.1211225986480713, 1.1209609508514404, 1.120215892791748, 1.180030345916748, 1.0977481603622437, 1.1623682975769043, 1.1419588327407837, 1.134534478187561, 1.1999119520187378, 1.1436896324157715, 1.1364456415176392, 1.14200758934021, 1.1159462928771973]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4052, "test_loss": 1.7079228759765626}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4102, "test_loss": 1.6998028198242188}], "final": {"test_acc": 0.4102, "test_loss": 1.6998028198242188, "wall_s": 47.32155680656433, "cos_r_negg": [0.8053134083747864, 0.3481922447681427, 0.289498895406723, 0.32326728105545044, 0.31446772813796997, 0.36413002014160156, 0.3290978968143463, 0.3191995620727539, 0.268321692943573, 0.37887489795684814, 0.22668762505054474, 0.26199910044670105, 0.361004114151001, 0.2945469319820404, 0.3109915256500244, 0.3107643723487854, 0.4017493426799774, 0.31529366970062256, 0.33951982855796814, 0.25017818808555603, 0.3865218162536621, 0.29690682888031006, 0.39198780059814453, 0.33076953887939453, 0.3715733289718628, 0.3544769883155823, 0.38510841131210327, 0.2954813539981842, 0.3582514524459839, 0.2942262589931488], "cos_innovation_negg": [0.8053134083747864, 0.3481922447681427, 0.289498895406723, 0.32326728105545044, 0.31446772813796997, 0.36413002014160156, 0.3290978968143463, 0.3191995620727539, 0.268321692943573, 0.37887489795684814, 0.22668762505054474, 0.26199910044670105, 0.361004114151001, 0.2945469319820404, 0.3109915256500244, 0.3107643723487854, 0.4017493426799774, 0.31529366970062256, 0.33951982855796814, 0.25017818808555603, 0.3865218162536621, 0.29690682888031006, 0.39198780059814453, 0.33076953887939453, 0.3715733289718628, 0.3544769883155823, 0.38510841131210327, 0.2954813539981842, 0.3582514524459839, 0.2942262589931488], "cos_apical_negg": [0.8053134083747864, 0.3481922447681427, 0.289498895406723, 0.32326728105545044, 0.31446772813796997, 0.36413002014160156, 0.3290978968143463, 0.3191995620727539, 0.268321692943573, 0.37887489795684814, 0.22668762505054474, 0.26199910044670105, 0.361004114151001, 0.2945469319820404, 0.3109915256500244, 0.3107643723487854, 0.4017493426799774, 0.31529366970062256, 0.33951982855796814, 0.25017818808555603, 0.3865218162536621, 0.29690682888031006, 0.39198780059814453, 0.33076953887939453, 0.3715733289718628, 0.3544769883155823, 0.38510841131210327, 0.2954813539981842, 0.3582514524459839, 0.2942262589931488], "cos_Ac_negg": [0.8053134083747864, 0.3481922447681427, 0.289498895406723, 0.32326728105545044, 0.31446772813796997, 0.36413002014160156, 0.3290978968143463, 0.3191995620727539, 0.268321692943573, 0.37887489795684814, 0.22668762505054474, 0.26199910044670105, 0.361004114151001, 0.2945469319820404, 0.3109915256500244, 0.3107643723487854, 0.4017493426799774, 0.31529366970062256, 0.33951982855796814, 0.25017818808555603, 0.3865218162536621, 0.29690682888031006, 0.39198780059814453, 0.33076953887939453, 0.3715733289718628, 0.3544769883155823, 0.38510841131210327, 0.2954813539981842, 0.3582514524459839, 0.2942262589931488]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d30_s3.json b/results/scale_v3_cifar10_sdil_w64_d30_s3.json new file mode 100644 index 0000000..055585c --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d30_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d30_s3", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.614835739135742, "cos_r_negg": [-0.03654135391116142, 0.0077547538094222546, -0.030528809875249863, -0.05135681852698326, 0.021922342479228973, -0.034611281007528305, 0.012567494064569473, -0.03248821198940277, 0.03575190156698227, 0.0626281350851059, -0.013408164493739605, 0.05921158939599991, 0.074644535779953, -0.02847941592335701, -0.03897911310195923, 0.11481612920761108, -0.02839096263051033, 0.03633828088641167, 0.009800894185900688, -0.0041430615819990635, 0.08480595052242279, -0.03772366791963577, -0.041099533438682556, -0.008383028209209442, 0.040631186217069626, -0.042176082730293274, 0.012553496286273003, 0.013110419735312462, -0.0015151642728596926, 0.06352084130048752], "cos_innovation_negg": [-0.03654135391116142, 0.0077547538094222546, -0.030528809875249863, -0.05135681852698326, 0.021922342479228973, -0.034611281007528305, 0.012567494064569473, -0.03248821198940277, 0.03575190156698227, 0.0626281350851059, -0.013408164493739605, 0.05921158939599991, 0.074644535779953, -0.02847941592335701, -0.03897911310195923, 0.11481612920761108, -0.02839096263051033, 0.03633828088641167, 0.009800894185900688, -0.0041430615819990635, 0.08480595052242279, -0.03772366791963577, -0.041099533438682556, -0.008383028209209442, 0.040631186217069626, -0.042176082730293274, 0.012553496286273003, 0.013110419735312462, -0.0015151642728596926, 0.06352084130048752], "cos_apical_negg": [-0.03654135391116142, 0.0077547538094222546, -0.030528809875249863, -0.05135681852698326, 0.021922342479228973, -0.034611281007528305, 0.012567494064569473, -0.03248821198940277, 0.03575190156698227, 0.0626281350851059, -0.013408164493739605, 0.05921158939599991, 0.074644535779953, -0.02847941592335701, -0.03897911310195923, 0.11481612920761108, -0.02839096263051033, 0.03633828088641167, 0.009800894185900688, -0.0041430615819990635, 0.08480595052242279, -0.03772366791963577, -0.041099533438682556, -0.008383028209209442, 0.040631186217069626, -0.042176082730293274, 0.012553496286273003, 0.013110419735312462, -0.0015151642728596926, 0.06352084130048752], "cos_Ac_negg": [-0.03654135391116142, 0.0077547538094222546, -0.030528809875249863, -0.05135681852698326, 0.021922342479228973, -0.034611281007528305, 0.012567494064569473, -0.03248821198940277, 0.03575190156698227, 0.0626281350851059, -0.013408164493739605, 0.05921158939599991, 0.074644535779953, -0.02847941592335701, -0.03897911310195923, 0.11481612920761108, -0.02839096263051033, 0.03633828088641167, 0.009800894185900688, -0.0041430615819990635, 0.08480595052242279, -0.03772366791963577, -0.041099533438682556, -0.008383028209209442, 0.040631186217069626, -0.042176082730293274, 0.012553496286273003, 0.013110419735312462, -0.0015151642728596926, 0.06352084130048752], "r_norm": [2.448432207107544, 2.3570010662078857, 2.6353750228881836, 2.430379867553711, 2.3661746978759766, 2.488997459411621, 2.524275302886963, 2.501565933227539, 2.5146260261535645, 2.4825754165649414, 2.571186065673828, 2.502138614654541, 2.371906042098999, 2.449497699737549, 2.3830647468566895, 2.442537784576416, 2.5044970512390137, 2.451873302459717, 2.5285000801086426, 2.466948986053467, 2.5582797527313232, 2.5348639488220215, 2.4186367988586426, 2.489137649536133, 2.4613142013549805, 2.435349941253662, 2.4119343757629395, 2.5761075019836426, 2.5193071365356445, 2.423402786254883], "ldr": {"dL_sdil": -0.12561702728271484, "dL_bp": -0.4997596740722656, "ratio": 0.2513548687494752}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3708, "test_loss": 1.7939301635742186}, {"step": 500, "epoch": 1, "train_loss": 1.6610409021377563, "cos_r_negg": [0.6486786007881165, 0.06922651827335358, 0.02717120200395584, 0.0008010598830878735, 0.09938080608844757, 0.05479602515697479, 0.02519191987812519, 0.024162722751498222, 0.04349470138549805, 0.11299987137317657, 0.044039689004421234, 0.1235000342130661, 0.01341967098414898, 0.15832899510860443, 0.08413100242614746, 0.132417231798172, 0.002664318773895502, 0.07572779059410095, 0.01202140748500824, 0.08049112558364868, 0.11604417115449905, 0.04773147404193878, 0.0918438583612442, -0.012441093102097511, 0.07387461513280869, 0.024333441630005836, 0.03603614866733551, 0.08734137564897537, 0.030478667467832565, 0.014134161174297333], "cos_innovation_negg": [0.6486786007881165, 0.06922651827335358, 0.02717120200395584, 0.0008010598830878735, 0.09938080608844757, 0.05479602515697479, 0.02519191987812519, 0.024162722751498222, 0.04349470138549805, 0.11299987137317657, 0.044039689004421234, 0.1235000342130661, 0.01341967098414898, 0.15832899510860443, 0.08413100242614746, 0.132417231798172, 0.002664318773895502, 0.07572779059410095, 0.01202140748500824, 0.08049112558364868, 0.11604417115449905, 0.04773147404193878, 0.0918438583612442, -0.012441093102097511, 0.07387461513280869, 0.024333441630005836, 0.03603614866733551, 0.08734137564897537, 0.030478667467832565, 0.014134161174297333], "cos_apical_negg": [0.6486786007881165, 0.06922651827335358, 0.02717120200395584, 0.0008010598830878735, 0.09938080608844757, 0.05479602515697479, 0.02519191987812519, 0.024162722751498222, 0.04349470138549805, 0.11299987137317657, 0.044039689004421234, 0.1235000342130661, 0.01341967098414898, 0.15832899510860443, 0.08413100242614746, 0.132417231798172, 0.002664318773895502, 0.07572779059410095, 0.01202140748500824, 0.08049112558364868, 0.11604417115449905, 0.04773147404193878, 0.0918438583612442, -0.012441093102097511, 0.07387461513280869, 0.024333441630005836, 0.03603614866733551, 0.08734137564897537, 0.030478667467832565, 0.014134161174297333], "cos_Ac_negg": [0.6486786007881165, 0.06922651827335358, 0.02717120200395584, 0.0008010598830878735, 0.09938080608844757, 0.05479602515697479, 0.02519191987812519, 0.024162722751498222, 0.04349470138549805, 0.11299987137317657, 0.044039689004421234, 0.1235000342130661, 0.01341967098414898, 0.15832899510860443, 0.08413100242614746, 0.132417231798172, 0.002664318773895502, 0.07572779059410095, 0.01202140748500824, 0.08049112558364868, 0.11604417115449905, 0.04773147404193878, 0.0918438583612442, -0.012441093102097511, 0.07387461513280869, 0.024333441630005836, 0.03603614866733551, 0.08734137564897537, 0.030478667467832565, 0.014134161174297333], "r_norm": [1.7434971332550049, 1.622157335281372, 1.795485019683838, 1.6583235263824463, 1.6536831855773926, 1.7115435600280762, 1.7459558248519897, 1.7406704425811768, 1.7445993423461914, 1.7272742986679077, 1.730987548828125, 1.730594515800476, 1.6298766136169434, 1.7348101139068604, 1.6295182704925537, 1.6809241771697998, 1.725515365600586, 1.6860625743865967, 1.7303556203842163, 1.7102277278900146, 1.7707462310791016, 1.727948546409607, 1.6421204805374146, 1.7073373794555664, 1.7121915817260742, 1.6815996170043945, 1.6404937505722046, 1.7911489009857178, 1.7258074283599854, 1.662035584449768]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3959, "test_loss": 1.7499875854492188}, {"step": 1000, "epoch": 2, "train_loss": 1.7586705684661865, "cos_r_negg": [0.734160304069519, 0.11674520373344421, 0.11126844584941864, 0.09968191385269165, 0.15611280500888824, 0.11370860040187836, 0.07758008688688278, 0.08192258328199387, 0.15560008585453033, 0.16929106414318085, 0.07465305924415588, 0.17864879965782166, 0.13104873895645142, 0.21081921458244324, 0.0925368219614029, 0.1830596625804901, 0.08066708594560623, 0.13731560111045837, 0.09163504093885422, 0.14182260632514954, 0.18179933726787567, 0.15615972876548767, 0.1484493613243103, 0.07618466764688492, 0.1452677696943283, 0.10690845549106598, 0.12238641083240509, 0.18829631805419922, 0.09195753931999207, 0.06697805225849152], "cos_innovation_negg": [0.734160304069519, 0.11674520373344421, 0.11126844584941864, 0.09968191385269165, 0.15611280500888824, 0.11370860040187836, 0.07758008688688278, 0.08192258328199387, 0.15560008585453033, 0.16929106414318085, 0.07465305924415588, 0.17864879965782166, 0.13104873895645142, 0.21081921458244324, 0.0925368219614029, 0.1830596625804901, 0.08066708594560623, 0.13731560111045837, 0.09163504093885422, 0.14182260632514954, 0.18179933726787567, 0.15615972876548767, 0.1484493613243103, 0.07618466764688492, 0.1452677696943283, 0.10690845549106598, 0.12238641083240509, 0.18829631805419922, 0.09195753931999207, 0.06697805225849152], "cos_apical_negg": [0.734160304069519, 0.11674520373344421, 0.11126844584941864, 0.09968191385269165, 0.15611280500888824, 0.11370860040187836, 0.07758008688688278, 0.08192258328199387, 0.15560008585453033, 0.16929106414318085, 0.07465305924415588, 0.17864879965782166, 0.13104873895645142, 0.21081921458244324, 0.0925368219614029, 0.1830596625804901, 0.08066708594560623, 0.13731560111045837, 0.09163504093885422, 0.14182260632514954, 0.18179933726787567, 0.15615972876548767, 0.1484493613243103, 0.07618466764688492, 0.1452677696943283, 0.10690845549106598, 0.12238641083240509, 0.18829631805419922, 0.09195753931999207, 0.06697805225849152], "cos_Ac_negg": [0.734160304069519, 0.11674520373344421, 0.11126844584941864, 0.09968191385269165, 0.15611280500888824, 0.11370860040187836, 0.07758008688688278, 0.08192258328199387, 0.15560008585453033, 0.16929106414318085, 0.07465305924415588, 0.17864879965782166, 0.13104873895645142, 0.21081921458244324, 0.0925368219614029, 0.1830596625804901, 0.08066708594560623, 0.13731560111045837, 0.09163504093885422, 0.14182260632514954, 0.18179933726787567, 0.15615972876548767, 0.1484493613243103, 0.07618466764688492, 0.1452677696943283, 0.10690845549106598, 0.12238641083240509, 0.18829631805419922, 0.09195753931999207, 0.06697805225849152], "r_norm": [1.489790916442871, 1.3102662563323975, 1.4456722736358643, 1.331455945968628, 1.3349518775939941, 1.3776518106460571, 1.4061938524246216, 1.396479606628418, 1.4072552919387817, 1.3990159034729004, 1.3965508937835693, 1.4076292514801025, 1.3029398918151855, 1.402430534362793, 1.3175594806671143, 1.3701577186584473, 1.3848907947540283, 1.3647581338882446, 1.3902668952941895, 1.373766303062439, 1.4394443035125732, 1.396249771118164, 1.3266220092773438, 1.3606624603271484, 1.3849308490753174, 1.3508663177490234, 1.318225622177124, 1.453971028327942, 1.3886768817901611, 1.3270347118377686]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.403, "test_loss": 1.7179084594726564}, {"step": 1500, "epoch": 3, "train_loss": 1.6222591400146484, "cos_r_negg": [0.7540536522865295, 0.22058254480361938, 0.22496911883354187, 0.1826854646205902, 0.2686963379383087, 0.20793196558952332, 0.18631722033023834, 0.21032854914665222, 0.2527831792831421, 0.23960039019584656, 0.1780945062637329, 0.30199551582336426, 0.26922136545181274, 0.3254665732383728, 0.223891019821167, 0.2788422405719757, 0.170053631067276, 0.2836529612541199, 0.19690042734146118, 0.23996244370937347, 0.241448312997818, 0.2200710028409958, 0.25812673568725586, 0.1727759689092636, 0.2632231116294861, 0.22593756020069122, 0.24917525053024292, 0.2687683701515198, 0.16965806484222412, 0.20134933292865753], "cos_innovation_negg": [0.7540536522865295, 0.22058254480361938, 0.22496911883354187, 0.1826854646205902, 0.2686963379383087, 0.20793196558952332, 0.18631722033023834, 0.21032854914665222, 0.2527831792831421, 0.23960039019584656, 0.1780945062637329, 0.30199551582336426, 0.26922136545181274, 0.3254665732383728, 0.223891019821167, 0.2788422405719757, 0.170053631067276, 0.2836529612541199, 0.19690042734146118, 0.23996244370937347, 0.241448312997818, 0.2200710028409958, 0.25812673568725586, 0.1727759689092636, 0.2632231116294861, 0.22593756020069122, 0.24917525053024292, 0.2687683701515198, 0.16965806484222412, 0.20134933292865753], "cos_apical_negg": [0.7540536522865295, 0.22058254480361938, 0.22496911883354187, 0.1826854646205902, 0.2686963379383087, 0.20793196558952332, 0.18631722033023834, 0.21032854914665222, 0.2527831792831421, 0.23960039019584656, 0.1780945062637329, 0.30199551582336426, 0.26922136545181274, 0.3254665732383728, 0.223891019821167, 0.2788422405719757, 0.170053631067276, 0.2836529612541199, 0.19690042734146118, 0.23996244370937347, 0.241448312997818, 0.2200710028409958, 0.25812673568725586, 0.1727759689092636, 0.2632231116294861, 0.22593756020069122, 0.24917525053024292, 0.2687683701515198, 0.16965806484222412, 0.20134933292865753], "cos_Ac_negg": [0.7540536522865295, 0.22058254480361938, 0.22496911883354187, 0.1826854646205902, 0.2686963379383087, 0.20793196558952332, 0.18631722033023834, 0.21032854914665222, 0.2527831792831421, 0.23960039019584656, 0.1780945062637329, 0.30199551582336426, 0.26922136545181274, 0.3254665732383728, 0.223891019821167, 0.2788422405719757, 0.170053631067276, 0.2836529612541199, 0.19690042734146118, 0.23996244370937347, 0.241448312997818, 0.2200710028409958, 0.25812673568725586, 0.1727759689092636, 0.2632231116294861, 0.22593756020069122, 0.24917525053024292, 0.2687683701515198, 0.16965806484222412, 0.20134933292865753], "r_norm": [1.3032163381576538, 1.073120355606079, 1.1824958324432373, 1.0922552347183228, 1.1043671369552612, 1.1308903694152832, 1.1548371315002441, 1.144289255142212, 1.1558849811553955, 1.1499626636505127, 1.145296573638916, 1.1724194288253784, 1.065477967262268, 1.1682496070861816, 1.0816391706466675, 1.1378849744796753, 1.1304028034210205, 1.1243600845336914, 1.1442229747772217, 1.1316503286361694, 1.192611813545227, 1.1536791324615479, 1.0962986946105957, 1.1052063703536987, 1.1378419399261475, 1.1200648546218872, 1.0859887599945068, 1.1951844692230225, 1.13640558719635, 1.0935431718826294]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4038, "test_loss": 1.71217724609375}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4198, "test_loss": 1.6856906127929687}], "final": {"test_acc": 0.4198, "test_loss": 1.6856906127929687, "wall_s": 49.04930329322815, "cos_r_negg": [0.7857164740562439, 0.3463512063026428, 0.2804408371448517, 0.2888534665107727, 0.3694070875644684, 0.30385634303092957, 0.281593382358551, 0.29843786358833313, 0.33403438329696655, 0.3499791622161865, 0.2845308184623718, 0.36064785718917847, 0.3142302632331848, 0.40371114015579224, 0.3318222165107727, 0.40019506216049194, 0.3049727976322174, 0.37598031759262085, 0.3199220299720764, 0.3436892330646515, 0.3364793062210083, 0.31414809823036194, 0.36200642585754395, 0.2813572585582733, 0.35465019941329956, 0.33925771713256836, 0.3480266332626343, 0.36071106791496277, 0.27095717191696167, 0.31454765796661377], "cos_innovation_negg": [0.7857164740562439, 0.3463512063026428, 0.2804408371448517, 0.2888534665107727, 0.3694070875644684, 0.30385634303092957, 0.281593382358551, 0.29843786358833313, 0.33403438329696655, 0.3499791622161865, 0.2845308184623718, 0.36064785718917847, 0.3142302632331848, 0.40371114015579224, 0.3318222165107727, 0.40019506216049194, 0.3049727976322174, 0.37598031759262085, 0.3199220299720764, 0.3436892330646515, 0.3364793062210083, 0.31414809823036194, 0.36200642585754395, 0.2813572585582733, 0.35465019941329956, 0.33925771713256836, 0.3480266332626343, 0.36071106791496277, 0.27095717191696167, 0.31454765796661377], "cos_apical_negg": [0.7857164740562439, 0.3463512063026428, 0.2804408371448517, 0.2888534665107727, 0.3694070875644684, 0.30385634303092957, 0.281593382358551, 0.29843786358833313, 0.33403438329696655, 0.3499791622161865, 0.2845308184623718, 0.36064785718917847, 0.3142302632331848, 0.40371114015579224, 0.3318222165107727, 0.40019506216049194, 0.3049727976322174, 0.37598031759262085, 0.3199220299720764, 0.3436892330646515, 0.3364793062210083, 0.31414809823036194, 0.36200642585754395, 0.2813572585582733, 0.35465019941329956, 0.33925771713256836, 0.3480266332626343, 0.36071106791496277, 0.27095717191696167, 0.31454765796661377], "cos_Ac_negg": [0.7857164740562439, 0.3463512063026428, 0.2804408371448517, 0.2888534665107727, 0.3694070875644684, 0.30385634303092957, 0.281593382358551, 0.29843786358833313, 0.33403438329696655, 0.3499791622161865, 0.2845308184623718, 0.36064785718917847, 0.3142302632331848, 0.40371114015579224, 0.3318222165107727, 0.40019506216049194, 0.3049727976322174, 0.37598031759262085, 0.3199220299720764, 0.3436892330646515, 0.3364793062210083, 0.31414809823036194, 0.36200642585754395, 0.2813572585582733, 0.35465019941329956, 0.33925771713256836, 0.3480266332626343, 0.36071106791496277, 0.27095717191696167, 0.31454765796661377]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d30_s4.json b/results/scale_v3_cifar10_sdil_w64_d30_s4.json new file mode 100644 index 0000000..3f71670 --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d30_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 30, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d30_s4", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6514735221862793, "cos_r_negg": [-0.09207373857498169, -0.040157608687877655, 0.02721032127737999, -0.06105586141347885, -0.0339331179857254, -0.06532096862792969, 0.010385488159954548, -0.036011289805173874, 0.0019292824435979128, 0.048837810754776, -0.042162612080574036, 0.02660932019352913, 0.12608708441257477, 0.014324294403195381, -0.025685761123895645, -0.018452057614922523, -0.041212718933820724, 0.023632187396287918, 0.037365347146987915, 0.0008449326269328594, -0.04360591247677803, 0.02498898282647133, -0.008458344265818596, 0.02910603955388069, 0.023649509996175766, 0.05854082107543945, -0.02538686990737915, -0.023190466687083244, -0.08888351917266846, -0.013678940013051033], "cos_innovation_negg": [-0.09207373857498169, -0.040157608687877655, 0.02721032127737999, -0.06105586141347885, -0.0339331179857254, -0.06532096862792969, 0.010385488159954548, -0.036011289805173874, 0.0019292824435979128, 0.048837810754776, -0.042162612080574036, 0.02660932019352913, 0.12608708441257477, 0.014324294403195381, -0.025685761123895645, -0.018452057614922523, -0.041212718933820724, 0.023632187396287918, 0.037365347146987915, 0.0008449326269328594, -0.04360591247677803, 0.02498898282647133, -0.008458344265818596, 0.02910603955388069, 0.023649509996175766, 0.05854082107543945, -0.02538686990737915, -0.023190466687083244, -0.08888351917266846, -0.013678940013051033], "cos_apical_negg": [-0.09207373857498169, -0.040157608687877655, 0.02721032127737999, -0.06105586141347885, -0.0339331179857254, -0.06532096862792969, 0.010385488159954548, -0.036011289805173874, 0.0019292824435979128, 0.048837810754776, -0.042162612080574036, 0.02660932019352913, 0.12608708441257477, 0.014324294403195381, -0.025685761123895645, -0.018452057614922523, -0.041212718933820724, 0.023632187396287918, 0.037365347146987915, 0.0008449326269328594, -0.04360591247677803, 0.02498898282647133, -0.008458344265818596, 0.02910603955388069, 0.023649509996175766, 0.05854082107543945, -0.02538686990737915, -0.023190466687083244, -0.08888351917266846, -0.013678940013051033], "cos_Ac_negg": [-0.09207373857498169, -0.040157608687877655, 0.02721032127737999, -0.06105586141347885, -0.0339331179857254, -0.06532096862792969, 0.010385488159954548, -0.036011289805173874, 0.0019292824435979128, 0.048837810754776, -0.042162612080574036, 0.02660932019352913, 0.12608708441257477, 0.014324294403195381, -0.025685761123895645, -0.018452057614922523, -0.041212718933820724, 0.023632187396287918, 0.037365347146987915, 0.0008449326269328594, -0.04360591247677803, 0.02498898282647133, -0.008458344265818596, 0.02910603955388069, 0.023649509996175766, 0.05854082107543945, -0.02538686990737915, -0.023190466687083244, -0.08888351917266846, -0.013678940013051033], "r_norm": [2.493131160736084, 2.500267505645752, 2.396979570388794, 2.5643973350524902, 2.416623115539551, 2.5048744678497314, 2.3001649379730225, 2.4149515628814697, 2.4704301357269287, 2.4743547439575195, 2.3493497371673584, 2.3925163745880127, 2.246513843536377, 2.383212089538574, 2.3734004497528076, 2.4716832637786865, 2.43491792678833, 2.447638750076294, 2.5086426734924316, 2.3625905513763428, 2.4226675033569336, 2.5421016216278076, 2.53977108001709, 2.3813443183898926, 2.5912532806396484, 2.587674617767334, 2.4324288368225098, 2.3443455696105957, 2.55172061920166, 2.4480018615722656], "ldr": {"dL_sdil": 0.14378929138183594, "dL_bp": -0.45875072479248047, "ratio": -0.3134366522186535}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3695, "test_loss": 1.8038250610351563}, {"step": 500, "epoch": 1, "train_loss": 1.6907622814178467, "cos_r_negg": [0.6043787002563477, -0.002317099366337061, 0.0430259108543396, 0.060774169862270355, 0.12662997841835022, 0.14248284697532654, 0.03823377192020416, 0.025301523506641388, 0.11370401829481125, 0.05965254828333855, 0.04358962923288345, 0.10678987205028534, 0.09511053562164307, 0.044847894459962845, 0.08666551113128662, 0.08298143744468689, 0.08376117050647736, 0.03060181997716427, 0.03665255755186081, 0.01682320050895214, 0.07931800186634064, 0.005747366696596146, 0.10371696949005127, 0.04708865284919739, 0.060145653784275055, -0.011076463386416435, 0.10610729455947876, 0.009080440737307072, -0.08103039115667343, 0.07445594668388367], "cos_innovation_negg": [0.6043787002563477, -0.002317099366337061, 0.0430259108543396, 0.060774169862270355, 0.12662997841835022, 0.14248284697532654, 0.03823377192020416, 0.025301523506641388, 0.11370401829481125, 0.05965254828333855, 0.04358962923288345, 0.10678987205028534, 0.09511053562164307, 0.044847894459962845, 0.08666551113128662, 0.08298143744468689, 0.08376117050647736, 0.03060181997716427, 0.03665255755186081, 0.01682320050895214, 0.07931800186634064, 0.005747366696596146, 0.10371696949005127, 0.04708865284919739, 0.060145653784275055, -0.011076463386416435, 0.10610729455947876, 0.009080440737307072, -0.08103039115667343, 0.07445594668388367], "cos_apical_negg": [0.6043787002563477, -0.002317099366337061, 0.0430259108543396, 0.060774169862270355, 0.12662997841835022, 0.14248284697532654, 0.03823377192020416, 0.025301523506641388, 0.11370401829481125, 0.05965254828333855, 0.04358962923288345, 0.10678987205028534, 0.09511053562164307, 0.044847894459962845, 0.08666551113128662, 0.08298143744468689, 0.08376117050647736, 0.03060181997716427, 0.03665255755186081, 0.01682320050895214, 0.07931800186634064, 0.005747366696596146, 0.10371696949005127, 0.04708865284919739, 0.060145653784275055, -0.011076463386416435, 0.10610729455947876, 0.009080440737307072, -0.08103039115667343, 0.07445594668388367], "cos_Ac_negg": [0.6043787002563477, -0.002317099366337061, 0.0430259108543396, 0.060774169862270355, 0.12662997841835022, 0.14248284697532654, 0.03823377192020416, 0.025301523506641388, 0.11370401829481125, 0.05965254828333855, 0.04358962923288345, 0.10678987205028534, 0.09511053562164307, 0.044847894459962845, 0.08666551113128662, 0.08298143744468689, 0.08376117050647736, 0.03060181997716427, 0.03665255755186081, 0.01682320050895214, 0.07931800186634064, 0.005747366696596146, 0.10371696949005127, 0.04708865284919739, 0.060145653784275055, -0.011076463386416435, 0.10610729455947876, 0.009080440737307072, -0.08103039115667343, 0.07445594668388367], "r_norm": [1.7601295709609985, 1.7281819581985474, 1.633253812789917, 1.7717325687408447, 1.644629955291748, 1.724133014678955, 1.5697107315063477, 1.6662423610687256, 1.7269182205200195, 1.6998569965362549, 1.6229617595672607, 1.646787405014038, 1.5729460716247559, 1.6328831911087036, 1.6359217166900635, 1.6931376457214355, 1.6869935989379883, 1.6773667335510254, 1.7101540565490723, 1.6387038230895996, 1.6584419012069702, 1.750970721244812, 1.7569489479064941, 1.6487245559692383, 1.780831217765808, 1.7728838920593262, 1.6737089157104492, 1.587500810623169, 1.743654727935791, 1.6770910024642944]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3953, "test_loss": 1.7469379150390625}, {"step": 1000, "epoch": 2, "train_loss": 1.6637003421783447, "cos_r_negg": [0.7126588821411133, 0.1087212860584259, 0.0962437093257904, 0.11991855502128601, 0.1812865138053894, 0.16881310939788818, 0.11489462852478027, 0.13917121291160583, 0.15665017068386078, 0.15146198868751526, 0.12530212104320526, 0.21093010902404785, 0.15561789274215698, 0.12909233570098877, 0.15520700812339783, 0.14885461330413818, 0.13689392805099487, 0.16058647632598877, 0.08171932399272919, 0.09557423740625381, 0.11722756177186966, 0.09906935691833496, 0.138742595911026, 0.12638792395591736, 0.1179674044251442, 0.05344969406723976, 0.16977614164352417, 0.0828913003206253, 0.05789879336953163, 0.15284490585327148], "cos_innovation_negg": [0.7126588821411133, 0.1087212860584259, 0.0962437093257904, 0.11991855502128601, 0.1812865138053894, 0.16881310939788818, 0.11489462852478027, 0.13917121291160583, 0.15665017068386078, 0.15146198868751526, 0.12530212104320526, 0.21093010902404785, 0.15561789274215698, 0.12909233570098877, 0.15520700812339783, 0.14885461330413818, 0.13689392805099487, 0.16058647632598877, 0.08171932399272919, 0.09557423740625381, 0.11722756177186966, 0.09906935691833496, 0.138742595911026, 0.12638792395591736, 0.1179674044251442, 0.05344969406723976, 0.16977614164352417, 0.0828913003206253, 0.05789879336953163, 0.15284490585327148], "cos_apical_negg": [0.7126588821411133, 0.1087212860584259, 0.0962437093257904, 0.11991855502128601, 0.1812865138053894, 0.16881310939788818, 0.11489462852478027, 0.13917121291160583, 0.15665017068386078, 0.15146198868751526, 0.12530212104320526, 0.21093010902404785, 0.15561789274215698, 0.12909233570098877, 0.15520700812339783, 0.14885461330413818, 0.13689392805099487, 0.16058647632598877, 0.08171932399272919, 0.09557423740625381, 0.11722756177186966, 0.09906935691833496, 0.138742595911026, 0.12638792395591736, 0.1179674044251442, 0.05344969406723976, 0.16977614164352417, 0.0828913003206253, 0.05789879336953163, 0.15284490585327148], "cos_Ac_negg": [0.7126588821411133, 0.1087212860584259, 0.0962437093257904, 0.11991855502128601, 0.1812865138053894, 0.16881310939788818, 0.11489462852478027, 0.13917121291160583, 0.15665017068386078, 0.15146198868751526, 0.12530212104320526, 0.21093010902404785, 0.15561789274215698, 0.12909233570098877, 0.15520700812339783, 0.14885461330413818, 0.13689392805099487, 0.16058647632598877, 0.08171932399272919, 0.09557423740625381, 0.11722756177186966, 0.09906935691833496, 0.138742595911026, 0.12638792395591736, 0.1179674044251442, 0.05344969406723976, 0.16977614164352417, 0.0828913003206253, 0.05789879336953163, 0.15284490585327148], "r_norm": [1.5390419960021973, 1.4361813068389893, 1.3597955703735352, 1.470674991607666, 1.3707301616668701, 1.4467016458511353, 1.308823823928833, 1.3826382160186768, 1.4431684017181396, 1.424412727355957, 1.3591235876083374, 1.3873100280761719, 1.316226601600647, 1.3631103038787842, 1.3659244775772095, 1.4084670543670654, 1.4000563621520996, 1.4025263786315918, 1.4272632598876953, 1.3568099737167358, 1.3828939199447632, 1.4455552101135254, 1.4682183265686035, 1.371239423751831, 1.482063889503479, 1.4641923904418945, 1.3988797664642334, 1.3175981044769287, 1.4366481304168701, 1.3956608772277832]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.395, "test_loss": 1.7369896362304686}, {"step": 1500, "epoch": 3, "train_loss": 1.6770312786102295, "cos_r_negg": [0.7336679100990295, 0.19703997671604156, 0.2089197039604187, 0.15830668807029724, 0.25051218271255493, 0.2477615922689438, 0.2646186947822571, 0.19741061329841614, 0.2559908628463745, 0.2379518449306488, 0.20128433406352997, 0.32477498054504395, 0.2508586645126343, 0.24413400888442993, 0.278148889541626, 0.246520534157753, 0.244541198015213, 0.21852168440818787, 0.1549578309059143, 0.16148683428764343, 0.23286423087120056, 0.21307730674743652, 0.20922528207302094, 0.21592850983142853, 0.2258681356906891, 0.17303495109081268, 0.2777699828147888, 0.23478707671165466, 0.1827113777399063, 0.2581007778644562], "cos_innovation_negg": [0.7336679100990295, 0.19703997671604156, 0.2089197039604187, 0.15830668807029724, 0.25051218271255493, 0.2477615922689438, 0.2646186947822571, 0.19741061329841614, 0.2559908628463745, 0.2379518449306488, 0.20128433406352997, 0.32477498054504395, 0.2508586645126343, 0.24413400888442993, 0.278148889541626, 0.246520534157753, 0.244541198015213, 0.21852168440818787, 0.1549578309059143, 0.16148683428764343, 0.23286423087120056, 0.21307730674743652, 0.20922528207302094, 0.21592850983142853, 0.2258681356906891, 0.17303495109081268, 0.2777699828147888, 0.23478707671165466, 0.1827113777399063, 0.2581007778644562], "cos_apical_negg": [0.7336679100990295, 0.19703997671604156, 0.2089197039604187, 0.15830668807029724, 0.25051218271255493, 0.2477615922689438, 0.2646186947822571, 0.19741061329841614, 0.2559908628463745, 0.2379518449306488, 0.20128433406352997, 0.32477498054504395, 0.2508586645126343, 0.24413400888442993, 0.278148889541626, 0.246520534157753, 0.244541198015213, 0.21852168440818787, 0.1549578309059143, 0.16148683428764343, 0.23286423087120056, 0.21307730674743652, 0.20922528207302094, 0.21592850983142853, 0.2258681356906891, 0.17303495109081268, 0.2777699828147888, 0.23478707671165466, 0.1827113777399063, 0.2581007778644562], "cos_Ac_negg": [0.7336679100990295, 0.19703997671604156, 0.2089197039604187, 0.15830668807029724, 0.25051218271255493, 0.2477615922689438, 0.2646186947822571, 0.19741061329841614, 0.2559908628463745, 0.2379518449306488, 0.20128433406352997, 0.32477498054504395, 0.2508586645126343, 0.24413400888442993, 0.278148889541626, 0.246520534157753, 0.244541198015213, 0.21852168440818787, 0.1549578309059143, 0.16148683428764343, 0.23286423087120056, 0.21307730674743652, 0.20922528207302094, 0.21592850983142853, 0.2258681356906891, 0.17303495109081268, 0.2777699828147888, 0.23478707671165466, 0.1827113777399063, 0.2581007778644562], "r_norm": [1.3230305910110474, 1.1652679443359375, 1.1062711477279663, 1.1897603273391724, 1.1224216222763062, 1.17976975440979, 1.067844033241272, 1.12227463722229, 1.1727925539016724, 1.1592687368392944, 1.1078935861587524, 1.140183925628662, 1.0714280605316162, 1.1174921989440918, 1.117173194885254, 1.149662971496582, 1.135904312133789, 1.1378989219665527, 1.1559553146362305, 1.096910834312439, 1.1268467903137207, 1.1743760108947754, 1.1925301551818848, 1.1087310314178467, 1.1930890083312988, 1.1767158508300781, 1.1370093822479248, 1.0678517818450928, 1.1574633121490479, 1.141463041305542]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4031, "test_loss": 1.7313075805664062}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4083, "test_loss": 1.7167727172851563}], "final": {"test_acc": 0.4083, "test_loss": 1.7167727172851563, "wall_s": 45.64832520484924, "cos_r_negg": [0.7377281785011292, 0.2985190749168396, 0.31252890825271606, 0.269855260848999, 0.3435976207256317, 0.3683837354183197, 0.3227652907371521, 0.3055039048194885, 0.36317145824432373, 0.3530462384223938, 0.3259979486465454, 0.42046475410461426, 0.3196040391921997, 0.3752676248550415, 0.3713603615760803, 0.3674395680427551, 0.3328475058078766, 0.31441059708595276, 0.28356456756591797, 0.2630889415740967, 0.3413650393486023, 0.3047230839729309, 0.32438334822654724, 0.3523517847061157, 0.3545776605606079, 0.3017430901527405, 0.3888435661792755, 0.32941627502441406, 0.30238470435142517, 0.38458311557769775], "cos_innovation_negg": [0.7377281785011292, 0.2985190749168396, 0.31252890825271606, 0.269855260848999, 0.3435976207256317, 0.3683837354183197, 0.3227652907371521, 0.3055039048194885, 0.36317145824432373, 0.3530462384223938, 0.3259979486465454, 0.42046475410461426, 0.3196040391921997, 0.3752676248550415, 0.3713603615760803, 0.3674395680427551, 0.3328475058078766, 0.31441059708595276, 0.28356456756591797, 0.2630889415740967, 0.3413650393486023, 0.3047230839729309, 0.32438334822654724, 0.3523517847061157, 0.3545776605606079, 0.3017430901527405, 0.3888435661792755, 0.32941627502441406, 0.30238470435142517, 0.38458311557769775], "cos_apical_negg": [0.7377281785011292, 0.2985190749168396, 0.31252890825271606, 0.269855260848999, 0.3435976207256317, 0.3683837354183197, 0.3227652907371521, 0.3055039048194885, 0.36317145824432373, 0.3530462384223938, 0.3259979486465454, 0.42046475410461426, 0.3196040391921997, 0.3752676248550415, 0.3713603615760803, 0.3674395680427551, 0.3328475058078766, 0.31441059708595276, 0.28356456756591797, 0.2630889415740967, 0.3413650393486023, 0.3047230839729309, 0.32438334822654724, 0.3523517847061157, 0.3545776605606079, 0.3017430901527405, 0.3888435661792755, 0.32941627502441406, 0.30238470435142517, 0.38458311557769775], "cos_Ac_negg": [0.7377281785011292, 0.2985190749168396, 0.31252890825271606, 0.269855260848999, 0.3435976207256317, 0.3683837354183197, 0.3227652907371521, 0.3055039048194885, 0.36317145824432373, 0.3530462384223938, 0.3259979486465454, 0.42046475410461426, 0.3196040391921997, 0.3752676248550415, 0.3713603615760803, 0.3674395680427551, 0.3328475058078766, 0.31441059708595276, 0.28356456756591797, 0.2630889415740967, 0.3413650393486023, 0.3047230839729309, 0.32438334822654724, 0.3523517847061157, 0.3545776605606079, 0.3017430901527405, 0.3888435661792755, 0.32941627502441406, 0.30238470435142517, 0.38458311557769775]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d60_s0.json b/results/scale_v3_cifar10_sdil_w64_d60_s0.json new file mode 100644 index 0000000..c75ab09 --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d60_s0.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 0, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d60_s0", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6154470443725586, "cos_r_negg": [0.00824391096830368, 0.03162052854895592, -0.047831855714321136, -0.0022021085023880005, -0.013032829388976097, -0.014932295307517052, -0.0704532340168953, -0.030040230602025986, -0.020447133108973503, -0.05151013284921646, 0.04950184002518654, -0.02089458890259266, 0.0030383728444576263, -0.05178888142108917, -0.05720045417547226, -0.06982016563415527, 0.08251547813415527, 0.04174470156431198, 0.05225787311792374, -0.03256798908114433, 0.023537758737802505, -0.008475148119032383, 0.07035811990499496, 0.018795045092701912, -0.008111847564578056, -0.011047604493796825, -0.06307624280452728, -0.004734567832201719, -0.0656220018863678, -0.0307687409222126, 0.0440501868724823, 0.004072677344083786, -0.02298356592655182, -0.032864805310964584, 0.0038536139763891697, 0.020486699417233467, -0.0055297925136983395, -0.0003991757985204458, 0.029928147792816162, 0.01716122217476368, 0.09282514452934265, 0.01995760202407837, -0.013349109329283237, 0.03773009404540062, -0.043992213904857635, 0.006984344683587551, -0.047033172100782394, -0.0005177173297852278, -0.05786652863025665, 0.04820503666996956, 0.005073794163763523, -0.10067319124937057, 0.030747149139642715, 0.029755573719739914, -0.01581287384033203, 0.042170196771621704, 0.013858259655535221, -0.06063992157578468, -0.048581261187791824, -0.031442247331142426], "cos_innovation_negg": [0.00824391096830368, 0.03162052854895592, -0.047831855714321136, -0.0022021085023880005, -0.013032829388976097, -0.014932295307517052, -0.0704532340168953, -0.030040230602025986, -0.020447133108973503, -0.05151013284921646, 0.04950184002518654, -0.02089458890259266, 0.0030383728444576263, -0.05178888142108917, -0.05720045417547226, -0.06982016563415527, 0.08251547813415527, 0.04174470156431198, 0.05225787311792374, -0.03256798908114433, 0.023537758737802505, -0.008475148119032383, 0.07035811990499496, 0.018795045092701912, -0.008111847564578056, -0.011047604493796825, -0.06307624280452728, -0.004734567832201719, -0.0656220018863678, -0.0307687409222126, 0.0440501868724823, 0.004072677344083786, -0.02298356592655182, -0.032864805310964584, 0.0038536139763891697, 0.020486699417233467, -0.0055297925136983395, -0.0003991757985204458, 0.029928147792816162, 0.01716122217476368, 0.09282514452934265, 0.01995760202407837, -0.013349109329283237, 0.03773009404540062, -0.043992213904857635, 0.006984344683587551, -0.047033172100782394, -0.0005177173297852278, -0.05786652863025665, 0.04820503666996956, 0.005073794163763523, -0.10067319124937057, 0.030747149139642715, 0.029755573719739914, -0.01581287384033203, 0.042170196771621704, 0.013858259655535221, -0.06063992157578468, -0.048581261187791824, -0.031442247331142426], "cos_apical_negg": [0.00824391096830368, 0.03162052854895592, -0.047831855714321136, -0.0022021085023880005, -0.013032829388976097, -0.014932295307517052, -0.0704532340168953, -0.030040230602025986, -0.020447133108973503, -0.05151013284921646, 0.04950184002518654, -0.02089458890259266, 0.0030383728444576263, -0.05178888142108917, -0.05720045417547226, -0.06982016563415527, 0.08251547813415527, 0.04174470156431198, 0.05225787311792374, -0.03256798908114433, 0.023537758737802505, -0.008475148119032383, 0.07035811990499496, 0.018795045092701912, -0.008111847564578056, -0.011047604493796825, -0.06307624280452728, -0.004734567832201719, -0.0656220018863678, -0.0307687409222126, 0.0440501868724823, 0.004072677344083786, -0.02298356592655182, -0.032864805310964584, 0.0038536139763891697, 0.020486699417233467, -0.0055297925136983395, -0.0003991757985204458, 0.029928147792816162, 0.01716122217476368, 0.09282514452934265, 0.01995760202407837, -0.013349109329283237, 0.03773009404540062, -0.043992213904857635, 0.006984344683587551, -0.047033172100782394, -0.0005177173297852278, -0.05786652863025665, 0.04820503666996956, 0.005073794163763523, -0.10067319124937057, 0.030747149139642715, 0.029755573719739914, -0.01581287384033203, 0.042170196771621704, 0.013858259655535221, -0.06063992157578468, -0.048581261187791824, -0.031442247331142426], "cos_Ac_negg": [0.00824391096830368, 0.03162052854895592, -0.047831855714321136, -0.0022021085023880005, -0.013032829388976097, -0.014932295307517052, -0.0704532340168953, -0.030040230602025986, -0.020447133108973503, -0.05151013284921646, 0.04950184002518654, -0.02089458890259266, 0.0030383728444576263, -0.05178888142108917, -0.05720045417547226, -0.06982016563415527, 0.08251547813415527, 0.04174470156431198, 0.05225787311792374, -0.03256798908114433, 0.023537758737802505, -0.008475148119032383, 0.07035811990499496, 0.018795045092701912, -0.008111847564578056, -0.011047604493796825, -0.06307624280452728, -0.004734567832201719, -0.0656220018863678, -0.0307687409222126, 0.0440501868724823, 0.004072677344083786, -0.02298356592655182, -0.032864805310964584, 0.0038536139763891697, 0.020486699417233467, -0.0055297925136983395, -0.0003991757985204458, 0.029928147792816162, 0.01716122217476368, 0.09282514452934265, 0.01995760202407837, -0.013349109329283237, 0.03773009404540062, -0.043992213904857635, 0.006984344683587551, -0.047033172100782394, -0.0005177173297852278, -0.05786652863025665, 0.04820503666996956, 0.005073794163763523, -0.10067319124937057, 0.030747149139642715, 0.029755573719739914, -0.01581287384033203, 0.042170196771621704, 0.013858259655535221, -0.06063992157578468, -0.048581261187791824, -0.031442247331142426], "r_norm": [2.475647211074829, 2.582141876220703, 2.5103986263275146, 2.4314181804656982, 2.278536796569824, 2.4668307304382324, 2.378121852874756, 2.4344351291656494, 2.5928449630737305, 2.5062460899353027, 2.404479742050171, 2.582801103591919, 2.477358102798462, 2.4683475494384766, 2.603311538696289, 2.509106159210205, 2.5784785747528076, 2.5190746784210205, 2.4874401092529297, 2.52705717086792, 2.6225593090057373, 2.4242684841156006, 2.4262771606445312, 2.5080440044403076, 2.4646711349487305, 2.4284114837646484, 2.6363186836242676, 2.448397636413574, 2.3711695671081543, 2.640806198120117, 2.3413212299346924, 2.4465043544769287, 2.4522440433502197, 2.473102569580078, 2.573472023010254, 2.523059129714966, 2.4016785621643066, 2.382770538330078, 2.5407238006591797, 2.457386016845703, 2.526772975921631, 2.433110237121582, 2.478888511657715, 2.427353858947754, 2.5112290382385254, 2.5137362480163574, 2.5532827377319336, 2.4558825492858887, 2.407565116882324, 2.4302077293395996, 2.3953957557678223, 2.5483880043029785, 2.3778128623962402, 2.4375057220458984, 2.5560481548309326, 2.3974103927612305, 2.608649253845215, 2.4859652519226074, 2.3996095657348633, 2.4269697666168213], "ldr": {"dL_sdil": 0.014339923858642578, "dL_bp": -0.4264340400695801, "ratio": -0.03362753089857173}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3654, "test_loss": 1.8139204467773438}, {"step": 500, "epoch": 1, "train_loss": 1.6714478731155396, "cos_r_negg": [0.6398711204528809, 0.08045336604118347, 0.06296725571155548, 0.03444591164588928, 0.04891926795244217, 0.09337839484214783, 0.06561367958784103, 0.07720770686864853, 0.06863469630479813, 0.02852531522512436, 0.0861290842294693, 0.021066123619675636, 0.06388528645038605, 0.033490002155303955, 0.06971554458141327, -0.002923880470916629, 0.038544632494449615, 0.06542747467756271, 0.1036529541015625, 0.0931016281247139, 0.024126440286636353, 0.10905695706605911, 0.12597110867500305, 0.09612717479467392, 0.12878364324569702, 0.032648757100105286, 0.07985586673021317, 0.01342830527573824, 0.017535505816340446, 0.0029379408806562424, 0.15186172723770142, 0.08372871577739716, 0.021085277199745178, 0.05216756463050842, 0.04087704420089722, 0.08477283269166946, 0.03431499004364014, 0.02604559063911438, 0.047012098133563995, 0.06130053848028183, 0.09004856646060944, 0.07965324074029922, 0.021382244303822517, 0.05597807094454765, 0.0441078245639801, 0.052947841584682465, 0.034906599670648575, 0.12976858019828796, -0.043593671172857285, 0.03311007469892502, -0.03262003883719444, 0.025063615292310715, 0.0022818464785814285, 0.09072045236825943, 0.17781250178813934, 0.1223243996500969, 0.11228570342063904, 0.1361311376094818, 0.002026174683123827, -0.003173322416841984], "cos_innovation_negg": [0.6398711204528809, 0.08045336604118347, 0.06296725571155548, 0.03444591164588928, 0.04891926795244217, 0.09337839484214783, 0.06561367958784103, 0.07720770686864853, 0.06863469630479813, 0.02852531522512436, 0.0861290842294693, 0.021066123619675636, 0.06388528645038605, 0.033490002155303955, 0.06971554458141327, -0.002923880470916629, 0.038544632494449615, 0.06542747467756271, 0.1036529541015625, 0.0931016281247139, 0.024126440286636353, 0.10905695706605911, 0.12597110867500305, 0.09612717479467392, 0.12878364324569702, 0.032648757100105286, 0.07985586673021317, 0.01342830527573824, 0.017535505816340446, 0.0029379408806562424, 0.15186172723770142, 0.08372871577739716, 0.021085277199745178, 0.05216756463050842, 0.04087704420089722, 0.08477283269166946, 0.03431499004364014, 0.02604559063911438, 0.047012098133563995, 0.06130053848028183, 0.09004856646060944, 0.07965324074029922, 0.021382244303822517, 0.05597807094454765, 0.0441078245639801, 0.052947841584682465, 0.034906599670648575, 0.12976858019828796, -0.043593671172857285, 0.03311007469892502, -0.03262003883719444, 0.025063615292310715, 0.0022818464785814285, 0.09072045236825943, 0.17781250178813934, 0.1223243996500969, 0.11228570342063904, 0.1361311376094818, 0.002026174683123827, -0.003173322416841984], "cos_apical_negg": [0.6398711204528809, 0.08045336604118347, 0.06296725571155548, 0.03444591164588928, 0.04891926795244217, 0.09337839484214783, 0.06561367958784103, 0.07720770686864853, 0.06863469630479813, 0.02852531522512436, 0.0861290842294693, 0.021066123619675636, 0.06388528645038605, 0.033490002155303955, 0.06971554458141327, -0.002923880470916629, 0.038544632494449615, 0.06542747467756271, 0.1036529541015625, 0.0931016281247139, 0.024126440286636353, 0.10905695706605911, 0.12597110867500305, 0.09612717479467392, 0.12878364324569702, 0.032648757100105286, 0.07985586673021317, 0.01342830527573824, 0.017535505816340446, 0.0029379408806562424, 0.15186172723770142, 0.08372871577739716, 0.021085277199745178, 0.05216756463050842, 0.04087704420089722, 0.08477283269166946, 0.03431499004364014, 0.02604559063911438, 0.047012098133563995, 0.06130053848028183, 0.09004856646060944, 0.07965324074029922, 0.021382244303822517, 0.05597807094454765, 0.0441078245639801, 0.052947841584682465, 0.034906599670648575, 0.12976858019828796, -0.043593671172857285, 0.03311007469892502, -0.03262003883719444, 0.025063615292310715, 0.0022818464785814285, 0.09072045236825943, 0.17781250178813934, 0.1223243996500969, 0.11228570342063904, 0.1361311376094818, 0.002026174683123827, -0.003173322416841984], "cos_Ac_negg": [0.6398711204528809, 0.08045336604118347, 0.06296725571155548, 0.03444591164588928, 0.04891926795244217, 0.09337839484214783, 0.06561367958784103, 0.07720770686864853, 0.06863469630479813, 0.02852531522512436, 0.0861290842294693, 0.021066123619675636, 0.06388528645038605, 0.033490002155303955, 0.06971554458141327, -0.002923880470916629, 0.038544632494449615, 0.06542747467756271, 0.1036529541015625, 0.0931016281247139, 0.024126440286636353, 0.10905695706605911, 0.12597110867500305, 0.09612717479467392, 0.12878364324569702, 0.032648757100105286, 0.07985586673021317, 0.01342830527573824, 0.017535505816340446, 0.0029379408806562424, 0.15186172723770142, 0.08372871577739716, 0.021085277199745178, 0.05216756463050842, 0.04087704420089722, 0.08477283269166946, 0.03431499004364014, 0.02604559063911438, 0.047012098133563995, 0.06130053848028183, 0.09004856646060944, 0.07965324074029922, 0.021382244303822517, 0.05597807094454765, 0.0441078245639801, 0.052947841584682465, 0.034906599670648575, 0.12976858019828796, -0.043593671172857285, 0.03311007469892502, -0.03262003883719444, 0.025063615292310715, 0.0022818464785814285, 0.09072045236825943, 0.17781250178813934, 0.1223243996500969, 0.11228570342063904, 0.1361311376094818, 0.002026174683123827, -0.003173322416841984], "r_norm": [1.744222640991211, 1.7673091888427734, 1.719620704650879, 1.6692619323730469, 1.5701978206634521, 1.6794977188110352, 1.619675874710083, 1.6615722179412842, 1.7946276664733887, 1.7021353244781494, 1.6710517406463623, 1.7518370151519775, 1.6914194822311401, 1.6794383525848389, 1.779116153717041, 1.7089316844940186, 1.7661702632904053, 1.7161190509796143, 1.6989675760269165, 1.7464306354522705, 1.796947717666626, 1.6674470901489258, 1.6811712980270386, 1.716463565826416, 1.6950021982192993, 1.6596821546554565, 1.8044252395629883, 1.6744993925094604, 1.6209625005722046, 1.7967761754989624, 1.5931060314178467, 1.6828787326812744, 1.6644529104232788, 1.6855206489562988, 1.757262945175171, 1.7377161979675293, 1.6286098957061768, 1.6447557210922241, 1.7378298044204712, 1.6871581077575684, 1.7478718757629395, 1.6712108850479126, 1.6990809440612793, 1.676346778869629, 1.7159614562988281, 1.719051480293274, 1.742910623550415, 1.693921446800232, 1.6481950283050537, 1.6715093851089478, 1.640086054801941, 1.7570922374725342, 1.6490960121154785, 1.6771860122680664, 1.7548454999923706, 1.6541526317596436, 1.7746543884277344, 1.7031097412109375, 1.6500136852264404, 1.6536366939544678]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3917, "test_loss": 1.7605324462890626}, {"step": 1000, "epoch": 2, "train_loss": 1.7386796474456787, "cos_r_negg": [0.7149659991264343, 0.1670122742652893, 0.09314488619565964, 0.11498928815126419, 0.12241406738758087, 0.12917405366897583, 0.11071436107158661, 0.1440773904323578, 0.1400883048772812, 0.1207822859287262, 0.14657053351402283, 0.07442326843738556, 0.14908245205879211, 0.06165171414613724, 0.13570082187652588, 0.03617776930332184, 0.07899796962738037, 0.14552965760231018, 0.15233905613422394, 0.18968290090560913, 0.09036578238010406, 0.15893997251987457, 0.1809094250202179, 0.15216019749641418, 0.198634535074234, 0.1756463348865509, 0.1495281159877777, 0.09190094470977783, 0.12810830771923065, 0.12022443860769272, 0.200030118227005, 0.17775483429431915, 0.07820558547973633, 0.11211375892162323, 0.12932491302490234, 0.20860761404037476, 0.1317947804927826, 0.08217453956604004, 0.09129787981510162, 0.12306429445743561, 0.16118690371513367, 0.1393539309501648, 0.09973818063735962, 0.13826069235801697, 0.09062434732913971, 0.11287806928157806, 0.14491289854049683, 0.18659758567810059, 0.05269630625844002, 0.061524514108896255, 0.09193314611911774, 0.10744161903858185, 0.04220355674624443, 0.16640610992908478, 0.22554731369018555, 0.15818336606025696, 0.11769498884677887, 0.16605177521705627, 0.0728539228439331, 0.08984527736902237], "cos_innovation_negg": [0.7149659991264343, 0.1670122742652893, 0.09314488619565964, 0.11498928815126419, 0.12241406738758087, 0.12917405366897583, 0.11071436107158661, 0.1440773904323578, 0.1400883048772812, 0.1207822859287262, 0.14657053351402283, 0.07442326843738556, 0.14908245205879211, 0.06165171414613724, 0.13570082187652588, 0.03617776930332184, 0.07899796962738037, 0.14552965760231018, 0.15233905613422394, 0.18968290090560913, 0.09036578238010406, 0.15893997251987457, 0.1809094250202179, 0.15216019749641418, 0.198634535074234, 0.1756463348865509, 0.1495281159877777, 0.09190094470977783, 0.12810830771923065, 0.12022443860769272, 0.200030118227005, 0.17775483429431915, 0.07820558547973633, 0.11211375892162323, 0.12932491302490234, 0.20860761404037476, 0.1317947804927826, 0.08217453956604004, 0.09129787981510162, 0.12306429445743561, 0.16118690371513367, 0.1393539309501648, 0.09973818063735962, 0.13826069235801697, 0.09062434732913971, 0.11287806928157806, 0.14491289854049683, 0.18659758567810059, 0.05269630625844002, 0.061524514108896255, 0.09193314611911774, 0.10744161903858185, 0.04220355674624443, 0.16640610992908478, 0.22554731369018555, 0.15818336606025696, 0.11769498884677887, 0.16605177521705627, 0.0728539228439331, 0.08984527736902237], "cos_apical_negg": [0.7149659991264343, 0.1670122742652893, 0.09314488619565964, 0.11498928815126419, 0.12241406738758087, 0.12917405366897583, 0.11071436107158661, 0.1440773904323578, 0.1400883048772812, 0.1207822859287262, 0.14657053351402283, 0.07442326843738556, 0.14908245205879211, 0.06165171414613724, 0.13570082187652588, 0.03617776930332184, 0.07899796962738037, 0.14552965760231018, 0.15233905613422394, 0.18968290090560913, 0.09036578238010406, 0.15893997251987457, 0.1809094250202179, 0.15216019749641418, 0.198634535074234, 0.1756463348865509, 0.1495281159877777, 0.09190094470977783, 0.12810830771923065, 0.12022443860769272, 0.200030118227005, 0.17775483429431915, 0.07820558547973633, 0.11211375892162323, 0.12932491302490234, 0.20860761404037476, 0.1317947804927826, 0.08217453956604004, 0.09129787981510162, 0.12306429445743561, 0.16118690371513367, 0.1393539309501648, 0.09973818063735962, 0.13826069235801697, 0.09062434732913971, 0.11287806928157806, 0.14491289854049683, 0.18659758567810059, 0.05269630625844002, 0.061524514108896255, 0.09193314611911774, 0.10744161903858185, 0.04220355674624443, 0.16640610992908478, 0.22554731369018555, 0.15818336606025696, 0.11769498884677887, 0.16605177521705627, 0.0728539228439331, 0.08984527736902237], "cos_Ac_negg": [0.7149659991264343, 0.1670122742652893, 0.09314488619565964, 0.11498928815126419, 0.12241406738758087, 0.12917405366897583, 0.11071436107158661, 0.1440773904323578, 0.1400883048772812, 0.1207822859287262, 0.14657053351402283, 0.07442326843738556, 0.14908245205879211, 0.06165171414613724, 0.13570082187652588, 0.03617776930332184, 0.07899796962738037, 0.14552965760231018, 0.15233905613422394, 0.18968290090560913, 0.09036578238010406, 0.15893997251987457, 0.1809094250202179, 0.15216019749641418, 0.198634535074234, 0.1756463348865509, 0.1495281159877777, 0.09190094470977783, 0.12810830771923065, 0.12022443860769272, 0.200030118227005, 0.17775483429431915, 0.07820558547973633, 0.11211375892162323, 0.12932491302490234, 0.20860761404037476, 0.1317947804927826, 0.08217453956604004, 0.09129787981510162, 0.12306429445743561, 0.16118690371513367, 0.1393539309501648, 0.09973818063735962, 0.13826069235801697, 0.09062434732913971, 0.11287806928157806, 0.14491289854049683, 0.18659758567810059, 0.05269630625844002, 0.061524514108896255, 0.09193314611911774, 0.10744161903858185, 0.04220355674624443, 0.16640610992908478, 0.22554731369018555, 0.15818336606025696, 0.11769498884677887, 0.16605177521705627, 0.0728539228439331, 0.08984527736902237], "r_norm": [1.5149874687194824, 1.4545912742614746, 1.410078763961792, 1.3641705513000488, 1.2916669845581055, 1.3796054124832153, 1.3321301937103271, 1.3659336566925049, 1.4815492630004883, 1.3868237733840942, 1.3695979118347168, 1.4372308254241943, 1.3968360424041748, 1.3742424249649048, 1.4629924297332764, 1.392016053199768, 1.4400757551193237, 1.410008192062378, 1.397935390472412, 1.4373369216918945, 1.4703164100646973, 1.3764104843139648, 1.3879746198654175, 1.4060325622558594, 1.4013479948043823, 1.3597780466079712, 1.4914779663085938, 1.3664891719818115, 1.3288205862045288, 1.4693293571472168, 1.315602421760559, 1.3788049221038818, 1.3658448457717896, 1.3820130825042725, 1.4370286464691162, 1.4375009536743164, 1.3476827144622803, 1.3393909931182861, 1.4190698862075806, 1.3837302923202515, 1.4398962259292603, 1.3666224479675293, 1.3899409770965576, 1.375233769416809, 1.4023609161376953, 1.4056668281555176, 1.42929208278656, 1.3997068405151367, 1.3415281772613525, 1.3628301620483398, 1.3313851356506348, 1.430267095565796, 1.3402422666549683, 1.3733458518981934, 1.4488221406936646, 1.3627090454101562, 1.458683967590332, 1.402953863143921, 1.350010633468628, 1.3495091199874878]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.4031, "test_loss": 1.7189087280273438}, {"step": 1500, "epoch": 3, "train_loss": 1.674216866493225, "cos_r_negg": [0.7562717795372009, 0.23670677840709686, 0.17480088770389557, 0.15152376890182495, 0.20917357504367828, 0.2414626032114029, 0.2050379365682602, 0.20528392493724823, 0.22965112328529358, 0.23302873969078064, 0.25278449058532715, 0.1806548833847046, 0.2373020350933075, 0.1885548084974289, 0.24202775955200195, 0.10442493855953217, 0.15905854105949402, 0.23355638980865479, 0.20943677425384521, 0.2593066692352295, 0.2014395296573639, 0.27389615774154663, 0.2723178565502167, 0.2842370867729187, 0.31301960349082947, 0.29388856887817383, 0.2478892207145691, 0.16036772727966309, 0.22407913208007812, 0.19607701897621155, 0.3110252618789673, 0.2773076593875885, 0.15167343616485596, 0.19562092423439026, 0.23808465898036957, 0.2769673466682434, 0.24957583844661713, 0.1914481818675995, 0.14318187534809113, 0.18917998671531677, 0.2828781306743622, 0.24087923765182495, 0.18703332543373108, 0.2174077033996582, 0.18807856738567352, 0.20496051013469696, 0.24633954465389252, 0.26210862398147583, 0.16335904598236084, 0.18114763498306274, 0.14670930802822113, 0.19647592306137085, 0.17429959774017334, 0.2453118860721588, 0.2778579890727997, 0.2434512823820114, 0.23584480583667755, 0.2681507468223572, 0.1965794414281845, 0.16891640424728394], "cos_innovation_negg": [0.7562717795372009, 0.23670677840709686, 0.17480088770389557, 0.15152376890182495, 0.20917357504367828, 0.2414626032114029, 0.2050379365682602, 0.20528392493724823, 0.22965112328529358, 0.23302873969078064, 0.25278449058532715, 0.1806548833847046, 0.2373020350933075, 0.1885548084974289, 0.24202775955200195, 0.10442493855953217, 0.15905854105949402, 0.23355638980865479, 0.20943677425384521, 0.2593066692352295, 0.2014395296573639, 0.27389615774154663, 0.2723178565502167, 0.2842370867729187, 0.31301960349082947, 0.29388856887817383, 0.2478892207145691, 0.16036772727966309, 0.22407913208007812, 0.19607701897621155, 0.3110252618789673, 0.2773076593875885, 0.15167343616485596, 0.19562092423439026, 0.23808465898036957, 0.2769673466682434, 0.24957583844661713, 0.1914481818675995, 0.14318187534809113, 0.18917998671531677, 0.2828781306743622, 0.24087923765182495, 0.18703332543373108, 0.2174077033996582, 0.18807856738567352, 0.20496051013469696, 0.24633954465389252, 0.26210862398147583, 0.16335904598236084, 0.18114763498306274, 0.14670930802822113, 0.19647592306137085, 0.17429959774017334, 0.2453118860721588, 0.2778579890727997, 0.2434512823820114, 0.23584480583667755, 0.2681507468223572, 0.1965794414281845, 0.16891640424728394], "cos_apical_negg": [0.7562717795372009, 0.23670677840709686, 0.17480088770389557, 0.15152376890182495, 0.20917357504367828, 0.2414626032114029, 0.2050379365682602, 0.20528392493724823, 0.22965112328529358, 0.23302873969078064, 0.25278449058532715, 0.1806548833847046, 0.2373020350933075, 0.1885548084974289, 0.24202775955200195, 0.10442493855953217, 0.15905854105949402, 0.23355638980865479, 0.20943677425384521, 0.2593066692352295, 0.2014395296573639, 0.27389615774154663, 0.2723178565502167, 0.2842370867729187, 0.31301960349082947, 0.29388856887817383, 0.2478892207145691, 0.16036772727966309, 0.22407913208007812, 0.19607701897621155, 0.3110252618789673, 0.2773076593875885, 0.15167343616485596, 0.19562092423439026, 0.23808465898036957, 0.2769673466682434, 0.24957583844661713, 0.1914481818675995, 0.14318187534809113, 0.18917998671531677, 0.2828781306743622, 0.24087923765182495, 0.18703332543373108, 0.2174077033996582, 0.18807856738567352, 0.20496051013469696, 0.24633954465389252, 0.26210862398147583, 0.16335904598236084, 0.18114763498306274, 0.14670930802822113, 0.19647592306137085, 0.17429959774017334, 0.2453118860721588, 0.2778579890727997, 0.2434512823820114, 0.23584480583667755, 0.2681507468223572, 0.1965794414281845, 0.16891640424728394], "cos_Ac_negg": [0.7562717795372009, 0.23670677840709686, 0.17480088770389557, 0.15152376890182495, 0.20917357504367828, 0.2414626032114029, 0.2050379365682602, 0.20528392493724823, 0.22965112328529358, 0.23302873969078064, 0.25278449058532715, 0.1806548833847046, 0.2373020350933075, 0.1885548084974289, 0.24202775955200195, 0.10442493855953217, 0.15905854105949402, 0.23355638980865479, 0.20943677425384521, 0.2593066692352295, 0.2014395296573639, 0.27389615774154663, 0.2723178565502167, 0.2842370867729187, 0.31301960349082947, 0.29388856887817383, 0.2478892207145691, 0.16036772727966309, 0.22407913208007812, 0.19607701897621155, 0.3110252618789673, 0.2773076593875885, 0.15167343616485596, 0.19562092423439026, 0.23808465898036957, 0.2769673466682434, 0.24957583844661713, 0.1914481818675995, 0.14318187534809113, 0.18917998671531677, 0.2828781306743622, 0.24087923765182495, 0.18703332543373108, 0.2174077033996582, 0.18807856738567352, 0.20496051013469696, 0.24633954465389252, 0.26210862398147583, 0.16335904598236084, 0.18114763498306274, 0.14670930802822113, 0.19647592306137085, 0.17429959774017334, 0.2453118860721588, 0.2778579890727997, 0.2434512823820114, 0.23584480583667755, 0.2681507468223572, 0.1965794414281845, 0.16891640424728394], "r_norm": [1.344200611114502, 1.2108817100524902, 1.167618751525879, 1.1232117414474487, 1.0768123865127563, 1.1558520793914795, 1.120426058769226, 1.1290935277938843, 1.2262012958526611, 1.1577458381652832, 1.1429945230484009, 1.1914457082748413, 1.1572247743606567, 1.1355586051940918, 1.2218539714813232, 1.145140528678894, 1.1937854290008545, 1.1785221099853516, 1.1596720218658447, 1.1984870433807373, 1.2131530046463013, 1.149593710899353, 1.1569311618804932, 1.1768555641174316, 1.1786514520645142, 1.137153148651123, 1.2441513538360596, 1.1271854639053345, 1.1030635833740234, 1.220018744468689, 1.1088409423828125, 1.1552425622940063, 1.1316726207733154, 1.1554527282714844, 1.1870787143707275, 1.2072536945343018, 1.1363164186477661, 1.1089236736297607, 1.1688015460968018, 1.1612458229064941, 1.2006111145019531, 1.1386542320251465, 1.151556134223938, 1.1441164016723633, 1.1658554077148438, 1.1673924922943115, 1.1913270950317383, 1.1675288677215576, 1.1059856414794922, 1.1245050430297852, 1.0982189178466797, 1.1864988803863525, 1.1021993160247803, 1.1495025157928467, 1.2244726419448853, 1.1306880712509155, 1.2213971614837646, 1.1756497621536255, 1.1182398796081543, 1.1142864227294922]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4072, "test_loss": 1.7166262573242188}, {"epoch_end": 4, "step": 1955, "test_acc": 0.411, "test_loss": 1.7010625}], "final": {"test_acc": 0.411, "test_loss": 1.7010625, "wall_s": 97.59175109863281, "cos_r_negg": [0.7643067836761475, 0.3249530792236328, 0.2837162911891937, 0.2855551838874817, 0.30676549673080444, 0.3069310188293457, 0.30197370052337646, 0.30546045303344727, 0.32763174176216125, 0.3318811058998108, 0.3541974425315857, 0.28489792346954346, 0.34482961893081665, 0.2764572501182556, 0.30864521861076355, 0.2257709801197052, 0.26245182752609253, 0.35390183329582214, 0.33345913887023926, 0.36278486251831055, 0.302906334400177, 0.3676672577857971, 0.3594485819339752, 0.3552466928958893, 0.4007866382598877, 0.33722585439682007, 0.34977155923843384, 0.2813144326210022, 0.3393842577934265, 0.2692403793334961, 0.3742133677005768, 0.3639702796936035, 0.25902092456817627, 0.296712189912796, 0.3503630757331848, 0.3699854612350464, 0.33875665068626404, 0.31278878450393677, 0.26691317558288574, 0.3222803473472595, 0.38388729095458984, 0.37448370456695557, 0.30006590485572815, 0.31624943017959595, 0.2847444713115692, 0.336752712726593, 0.33294251561164856, 0.3756948411464691, 0.2714354395866394, 0.28742074966430664, 0.27885672450065613, 0.3194308280944824, 0.28793802857398987, 0.3472953736782074, 0.36703982949256897, 0.3526768684387207, 0.35906413197517395, 0.37517479062080383, 0.28327423334121704, 0.3131701946258545], "cos_innovation_negg": [0.7643067836761475, 0.3249530792236328, 0.2837162911891937, 0.2855551838874817, 0.30676549673080444, 0.3069310188293457, 0.30197370052337646, 0.30546045303344727, 0.32763174176216125, 0.3318811058998108, 0.3541974425315857, 0.28489792346954346, 0.34482961893081665, 0.2764572501182556, 0.30864521861076355, 0.2257709801197052, 0.26245182752609253, 0.35390183329582214, 0.33345913887023926, 0.36278486251831055, 0.302906334400177, 0.3676672577857971, 0.3594485819339752, 0.3552466928958893, 0.4007866382598877, 0.33722585439682007, 0.34977155923843384, 0.2813144326210022, 0.3393842577934265, 0.2692403793334961, 0.3742133677005768, 0.3639702796936035, 0.25902092456817627, 0.296712189912796, 0.3503630757331848, 0.3699854612350464, 0.33875665068626404, 0.31278878450393677, 0.26691317558288574, 0.3222803473472595, 0.38388729095458984, 0.37448370456695557, 0.30006590485572815, 0.31624943017959595, 0.2847444713115692, 0.336752712726593, 0.33294251561164856, 0.3756948411464691, 0.2714354395866394, 0.28742074966430664, 0.27885672450065613, 0.3194308280944824, 0.28793802857398987, 0.3472953736782074, 0.36703982949256897, 0.3526768684387207, 0.35906413197517395, 0.37517479062080383, 0.28327423334121704, 0.3131701946258545], "cos_apical_negg": [0.7643067836761475, 0.3249530792236328, 0.2837162911891937, 0.2855551838874817, 0.30676549673080444, 0.3069310188293457, 0.30197370052337646, 0.30546045303344727, 0.32763174176216125, 0.3318811058998108, 0.3541974425315857, 0.28489792346954346, 0.34482961893081665, 0.2764572501182556, 0.30864521861076355, 0.2257709801197052, 0.26245182752609253, 0.35390183329582214, 0.33345913887023926, 0.36278486251831055, 0.302906334400177, 0.3676672577857971, 0.3594485819339752, 0.3552466928958893, 0.4007866382598877, 0.33722585439682007, 0.34977155923843384, 0.2813144326210022, 0.3393842577934265, 0.2692403793334961, 0.3742133677005768, 0.3639702796936035, 0.25902092456817627, 0.296712189912796, 0.3503630757331848, 0.3699854612350464, 0.33875665068626404, 0.31278878450393677, 0.26691317558288574, 0.3222803473472595, 0.38388729095458984, 0.37448370456695557, 0.30006590485572815, 0.31624943017959595, 0.2847444713115692, 0.336752712726593, 0.33294251561164856, 0.3756948411464691, 0.2714354395866394, 0.28742074966430664, 0.27885672450065613, 0.3194308280944824, 0.28793802857398987, 0.3472953736782074, 0.36703982949256897, 0.3526768684387207, 0.35906413197517395, 0.37517479062080383, 0.28327423334121704, 0.3131701946258545], "cos_Ac_negg": [0.7643067836761475, 0.3249530792236328, 0.2837162911891937, 0.2855551838874817, 0.30676549673080444, 0.3069310188293457, 0.30197370052337646, 0.30546045303344727, 0.32763174176216125, 0.3318811058998108, 0.3541974425315857, 0.28489792346954346, 0.34482961893081665, 0.2764572501182556, 0.30864521861076355, 0.2257709801197052, 0.26245182752609253, 0.35390183329582214, 0.33345913887023926, 0.36278486251831055, 0.302906334400177, 0.3676672577857971, 0.3594485819339752, 0.3552466928958893, 0.4007866382598877, 0.33722585439682007, 0.34977155923843384, 0.2813144326210022, 0.3393842577934265, 0.2692403793334961, 0.3742133677005768, 0.3639702796936035, 0.25902092456817627, 0.296712189912796, 0.3503630757331848, 0.3699854612350464, 0.33875665068626404, 0.31278878450393677, 0.26691317558288574, 0.3222803473472595, 0.38388729095458984, 0.37448370456695557, 0.30006590485572815, 0.31624943017959595, 0.2847444713115692, 0.336752712726593, 0.33294251561164856, 0.3756948411464691, 0.2714354395866394, 0.28742074966430664, 0.27885672450065613, 0.3194308280944824, 0.28793802857398987, 0.3472953736782074, 0.36703982949256897, 0.3526768684387207, 0.35906413197517395, 0.37517479062080383, 0.28327423334121704, 0.3131701946258545]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d60_s1.json b/results/scale_v3_cifar10_sdil_w64_d60_s1.json new file mode 100644 index 0000000..0594a9c --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d60_s1.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 1, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d60_s1", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.473468780517578, "cos_r_negg": [-0.035722892731428146, -0.014311438426375389, 0.04019647091627121, 0.0990113839507103, 0.05390103906393051, 0.009961535222828388, 0.04427438974380493, 0.004886348731815815, -0.04224563017487526, -0.028240147978067398, 0.018397528678178787, -0.015100765973329544, -0.036847516894340515, -0.011993798427283764, 0.03389846533536911, -0.004846538417041302, -0.04474658519029617, 0.04842512309551239, -5.9973448514938354e-05, 0.005345525220036507, -0.015117490664124489, 0.016512280330061913, 0.016087785363197327, -0.028662167489528656, -0.08204537630081177, 0.03667046129703522, -0.031052809208631516, -0.032636143267154694, -0.021091215312480927, 0.03389746695756912, -0.04122815281152725, 0.03931804001331329, -0.033466942608356476, 0.015912141650915146, 0.01132176723331213, 0.04170200601220131, -0.04419345781207085, -0.0015074710827320814, 0.07226534187793732, -0.03511534258723259, -0.020587267354130745, 0.010057584382593632, 0.03808574378490448, 0.006610808428376913, -0.017624471336603165, 0.03917095065116882, -0.056267037987709045, -0.0011651096865534782, 0.012098584324121475, -0.01895175129175186, -0.010964690707623959, 0.026341434568166733, 0.0382668673992157, -0.021695051342248917, 0.016682345420122147, -0.037257932126522064, 0.034343428909778595, -0.061707086861133575, 0.051681339740753174, -0.030312836170196533], "cos_innovation_negg": [-0.035722892731428146, -0.014311438426375389, 0.04019647091627121, 0.0990113839507103, 0.05390103906393051, 0.009961535222828388, 0.04427438974380493, 0.004886348731815815, -0.04224563017487526, -0.028240147978067398, 0.018397528678178787, -0.015100765973329544, -0.036847516894340515, -0.011993798427283764, 0.03389846533536911, -0.004846538417041302, -0.04474658519029617, 0.04842512309551239, -5.9973448514938354e-05, 0.005345525220036507, -0.015117490664124489, 0.016512280330061913, 0.016087785363197327, -0.028662167489528656, -0.08204537630081177, 0.03667046129703522, -0.031052809208631516, -0.032636143267154694, -0.021091215312480927, 0.03389746695756912, -0.04122815281152725, 0.03931804001331329, -0.033466942608356476, 0.015912141650915146, 0.01132176723331213, 0.04170200601220131, -0.04419345781207085, -0.0015074710827320814, 0.07226534187793732, -0.03511534258723259, -0.020587267354130745, 0.010057584382593632, 0.03808574378490448, 0.006610808428376913, -0.017624471336603165, 0.03917095065116882, -0.056267037987709045, -0.0011651096865534782, 0.012098584324121475, -0.01895175129175186, -0.010964690707623959, 0.026341434568166733, 0.0382668673992157, -0.021695051342248917, 0.016682345420122147, -0.037257932126522064, 0.034343428909778595, -0.061707086861133575, 0.051681339740753174, -0.030312836170196533], "cos_apical_negg": [-0.035722892731428146, -0.014311438426375389, 0.04019647091627121, 0.0990113839507103, 0.05390103906393051, 0.009961535222828388, 0.04427438974380493, 0.004886348731815815, -0.04224563017487526, -0.028240147978067398, 0.018397528678178787, -0.015100765973329544, -0.036847516894340515, -0.011993798427283764, 0.03389846533536911, -0.004846538417041302, -0.04474658519029617, 0.04842512309551239, -5.9973448514938354e-05, 0.005345525220036507, -0.015117490664124489, 0.016512280330061913, 0.016087785363197327, -0.028662167489528656, -0.08204537630081177, 0.03667046129703522, -0.031052809208631516, -0.032636143267154694, -0.021091215312480927, 0.03389746695756912, -0.04122815281152725, 0.03931804001331329, -0.033466942608356476, 0.015912141650915146, 0.01132176723331213, 0.04170200601220131, -0.04419345781207085, -0.0015074710827320814, 0.07226534187793732, -0.03511534258723259, -0.020587267354130745, 0.010057584382593632, 0.03808574378490448, 0.006610808428376913, -0.017624471336603165, 0.03917095065116882, -0.056267037987709045, -0.0011651096865534782, 0.012098584324121475, -0.01895175129175186, -0.010964690707623959, 0.026341434568166733, 0.0382668673992157, -0.021695051342248917, 0.016682345420122147, -0.037257932126522064, 0.034343428909778595, -0.061707086861133575, 0.051681339740753174, -0.030312836170196533], "cos_Ac_negg": [-0.035722892731428146, -0.014311438426375389, 0.04019647091627121, 0.0990113839507103, 0.05390103906393051, 0.009961535222828388, 0.04427438974380493, 0.004886348731815815, -0.04224563017487526, -0.028240147978067398, 0.018397528678178787, -0.015100765973329544, -0.036847516894340515, -0.011993798427283764, 0.03389846533536911, -0.004846538417041302, -0.04474658519029617, 0.04842512309551239, -5.9973448514938354e-05, 0.005345525220036507, -0.015117490664124489, 0.016512280330061913, 0.016087785363197327, -0.028662167489528656, -0.08204537630081177, 0.03667046129703522, -0.031052809208631516, -0.032636143267154694, -0.021091215312480927, 0.03389746695756912, -0.04122815281152725, 0.03931804001331329, -0.033466942608356476, 0.015912141650915146, 0.01132176723331213, 0.04170200601220131, -0.04419345781207085, -0.0015074710827320814, 0.07226534187793732, -0.03511534258723259, -0.020587267354130745, 0.010057584382593632, 0.03808574378490448, 0.006610808428376913, -0.017624471336603165, 0.03917095065116882, -0.056267037987709045, -0.0011651096865534782, 0.012098584324121475, -0.01895175129175186, -0.010964690707623959, 0.026341434568166733, 0.0382668673992157, -0.021695051342248917, 0.016682345420122147, -0.037257932126522064, 0.034343428909778595, -0.061707086861133575, 0.051681339740753174, -0.030312836170196533], "r_norm": [2.4107041358947754, 2.422424077987671, 2.34220290184021, 2.527778148651123, 2.2393105030059814, 2.4752790927886963, 2.494619131088257, 2.3315420150756836, 2.4779953956604004, 2.4662933349609375, 2.3924288749694824, 2.428877592086792, 2.4622106552124023, 2.361783504486084, 2.4296913146972656, 2.3216865062713623, 2.4634337425231934, 2.466848373413086, 2.470992088317871, 2.5634076595306396, 2.500354766845703, 2.483220100402832, 2.4131975173950195, 2.4562878608703613, 2.419389247894287, 2.357121706008911, 2.4092330932617188, 2.3891994953155518, 2.466686487197876, 2.421828269958496, 2.3516221046447754, 2.5187911987304688, 2.486713409423828, 2.3775057792663574, 2.5289711952209473, 2.292257308959961, 2.39833402633667, 2.378805160522461, 2.5517001152038574, 2.240609645843506, 2.437047243118286, 2.601186513900757, 2.442793846130371, 2.4335765838623047, 2.540621519088745, 2.3947248458862305, 2.3929710388183594, 2.3135571479797363, 2.442608594894409, 2.4056923389434814, 2.4013960361480713, 2.5072853565216064, 2.406625986099243, 2.426177501678467, 2.3819479942321777, 2.391493320465088, 2.568481206893921, 2.3945798873901367, 2.467118978500366, 2.57377290725708], "ldr": {"dL_sdil": 0.014344215393066406, "dL_bp": -0.2991161346435547, "ratio": -0.04795533818381233}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3648, "test_loss": 1.7986327758789062}, {"step": 500, "epoch": 1, "train_loss": 1.7513988018035889, "cos_r_negg": [0.6380303502082825, 0.008565405383706093, 0.05498768389225006, 0.0031249523162841797, 0.07353948056697845, 0.08869121968746185, 0.07864750921726227, 0.08173838257789612, 0.05198744311928749, 0.11396027356386185, 0.10753995180130005, 0.033096425235271454, 0.05114371329545975, 0.07623224705457687, 0.10011599212884903, 0.0550856851041317, 0.033869944512844086, 0.1125168651342392, 0.06662203371524811, 0.08315375447273254, 0.028134945780038834, 0.08585704863071442, 0.036690954118967056, 0.07889609038829803, 0.03996600955724716, 0.04385967180132866, 0.08443712443113327, 0.027814077213406563, 0.01076043862849474, 0.11723334342241287, 0.034871719777584076, 0.06682755053043365, 0.06838163733482361, 0.04464304819703102, 0.09220969676971436, 0.09072793275117874, 0.05262100324034691, -0.012398948892951012, 0.1379569172859192, 0.07112517952919006, 0.07018661499023438, 0.08338402211666107, 0.060128048062324524, 0.021178733557462692, 0.08113966882228851, 0.06822612881660461, 0.0265495628118515, 0.06754353642463684, 0.10096229612827301, 0.03714583441615105, 0.09133277833461761, 0.10457479953765869, 0.049985021352767944, 0.052503447979688644, 0.041957754641771317, -0.01411544531583786, 0.05572324991226196, 0.016709264367818832, 0.09585592895746231, -0.004803758580237627], "cos_innovation_negg": [0.6380303502082825, 0.008565405383706093, 0.05498768389225006, 0.0031249523162841797, 0.07353948056697845, 0.08869121968746185, 0.07864750921726227, 0.08173838257789612, 0.05198744311928749, 0.11396027356386185, 0.10753995180130005, 0.033096425235271454, 0.05114371329545975, 0.07623224705457687, 0.10011599212884903, 0.0550856851041317, 0.033869944512844086, 0.1125168651342392, 0.06662203371524811, 0.08315375447273254, 0.028134945780038834, 0.08585704863071442, 0.036690954118967056, 0.07889609038829803, 0.03996600955724716, 0.04385967180132866, 0.08443712443113327, 0.027814077213406563, 0.01076043862849474, 0.11723334342241287, 0.034871719777584076, 0.06682755053043365, 0.06838163733482361, 0.04464304819703102, 0.09220969676971436, 0.09072793275117874, 0.05262100324034691, -0.012398948892951012, 0.1379569172859192, 0.07112517952919006, 0.07018661499023438, 0.08338402211666107, 0.060128048062324524, 0.021178733557462692, 0.08113966882228851, 0.06822612881660461, 0.0265495628118515, 0.06754353642463684, 0.10096229612827301, 0.03714583441615105, 0.09133277833461761, 0.10457479953765869, 0.049985021352767944, 0.052503447979688644, 0.041957754641771317, -0.01411544531583786, 0.05572324991226196, 0.016709264367818832, 0.09585592895746231, -0.004803758580237627], "cos_apical_negg": [0.6380303502082825, 0.008565405383706093, 0.05498768389225006, 0.0031249523162841797, 0.07353948056697845, 0.08869121968746185, 0.07864750921726227, 0.08173838257789612, 0.05198744311928749, 0.11396027356386185, 0.10753995180130005, 0.033096425235271454, 0.05114371329545975, 0.07623224705457687, 0.10011599212884903, 0.0550856851041317, 0.033869944512844086, 0.1125168651342392, 0.06662203371524811, 0.08315375447273254, 0.028134945780038834, 0.08585704863071442, 0.036690954118967056, 0.07889609038829803, 0.03996600955724716, 0.04385967180132866, 0.08443712443113327, 0.027814077213406563, 0.01076043862849474, 0.11723334342241287, 0.034871719777584076, 0.06682755053043365, 0.06838163733482361, 0.04464304819703102, 0.09220969676971436, 0.09072793275117874, 0.05262100324034691, -0.012398948892951012, 0.1379569172859192, 0.07112517952919006, 0.07018661499023438, 0.08338402211666107, 0.060128048062324524, 0.021178733557462692, 0.08113966882228851, 0.06822612881660461, 0.0265495628118515, 0.06754353642463684, 0.10096229612827301, 0.03714583441615105, 0.09133277833461761, 0.10457479953765869, 0.049985021352767944, 0.052503447979688644, 0.041957754641771317, -0.01411544531583786, 0.05572324991226196, 0.016709264367818832, 0.09585592895746231, -0.004803758580237627], "cos_Ac_negg": [0.6380303502082825, 0.008565405383706093, 0.05498768389225006, 0.0031249523162841797, 0.07353948056697845, 0.08869121968746185, 0.07864750921726227, 0.08173838257789612, 0.05198744311928749, 0.11396027356386185, 0.10753995180130005, 0.033096425235271454, 0.05114371329545975, 0.07623224705457687, 0.10011599212884903, 0.0550856851041317, 0.033869944512844086, 0.1125168651342392, 0.06662203371524811, 0.08315375447273254, 0.028134945780038834, 0.08585704863071442, 0.036690954118967056, 0.07889609038829803, 0.03996600955724716, 0.04385967180132866, 0.08443712443113327, 0.027814077213406563, 0.01076043862849474, 0.11723334342241287, 0.034871719777584076, 0.06682755053043365, 0.06838163733482361, 0.04464304819703102, 0.09220969676971436, 0.09072793275117874, 0.05262100324034691, -0.012398948892951012, 0.1379569172859192, 0.07112517952919006, 0.07018661499023438, 0.08338402211666107, 0.060128048062324524, 0.021178733557462692, 0.08113966882228851, 0.06822612881660461, 0.0265495628118515, 0.06754353642463684, 0.10096229612827301, 0.03714583441615105, 0.09133277833461761, 0.10457479953765869, 0.049985021352767944, 0.052503447979688644, 0.041957754641771317, -0.01411544531583786, 0.05572324991226196, 0.016709264367818832, 0.09585592895746231, -0.004803758580237627], "r_norm": [1.7633413076400757, 1.6989986896514893, 1.6543883085250854, 1.7922239303588867, 1.601600170135498, 1.7519359588623047, 1.761155605316162, 1.6363821029663086, 1.7400013208389282, 1.7420713901519775, 1.6764302253723145, 1.7082949876785278, 1.7343026399612427, 1.6796252727508545, 1.718650460243225, 1.6357779502868652, 1.729400396347046, 1.7485625743865967, 1.7270565032958984, 1.8156522512435913, 1.7527098655700684, 1.7545747756958008, 1.7048221826553345, 1.73529052734375, 1.6877433061599731, 1.6694716215133667, 1.7093764543533325, 1.6722829341888428, 1.7311817407608032, 1.7190511226654053, 1.6541807651519775, 1.777063012123108, 1.7468960285186768, 1.6697196960449219, 1.7832844257354736, 1.630014419555664, 1.692211389541626, 1.6573530435562134, 1.8209514617919922, 1.5693937540054321, 1.7247748374938965, 1.8464256525039673, 1.7357524633407593, 1.7130248546600342, 1.8055775165557861, 1.6837512254714966, 1.6802853345870972, 1.6450300216674805, 1.7314196825027466, 1.6911810636520386, 1.706683874130249, 1.7662060260772705, 1.7019901275634766, 1.7231684923171997, 1.6855347156524658, 1.6748408079147339, 1.813249111175537, 1.6893904209136963, 1.745285153388977, 1.8135223388671875]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3934, "test_loss": 1.7465070556640625}, {"step": 1000, "epoch": 2, "train_loss": 1.7546114921569824, "cos_r_negg": [0.7281479239463806, 0.12392101436853409, 0.09547808766365051, 0.09054690599441528, 0.12905195355415344, 0.14604750275611877, 0.15337306261062622, 0.12187179177999496, 0.09774170815944672, 0.13584667444229126, 0.21536147594451904, 0.1164068728685379, 0.12027052044868469, 0.10585986077785492, 0.1531606912612915, 0.07483823597431183, 0.11418572813272476, 0.1420007050037384, 0.11972622573375702, 0.1401602178812027, 0.15721440315246582, 0.15242041647434235, 0.1132228672504425, 0.12021379917860031, 0.11057637631893158, 0.15405189990997314, 0.14090479910373688, 0.06371747702360153, 0.059767965227365494, 0.18710459768772125, 0.10280166566371918, 0.07482711970806122, 0.08285433799028397, 0.18478065729141235, 0.08931631594896317, 0.12666559219360352, 0.14000849425792694, 0.019589923322200775, 0.16820503771305084, 0.13894467055797577, 0.0829760879278183, 0.07348090410232544, 0.13687270879745483, 0.06221502274274826, 0.12125816196203232, 0.135206937789917, 0.08886708319187164, 0.07825340330600739, 0.22789394855499268, 0.14402498304843903, 0.17377740144729614, 0.1298290193080902, 0.14159110188484192, 0.14188897609710693, 0.1107911467552185, 0.07106991857290268, 0.10498370230197906, 0.10174622386693954, 0.14011073112487793, 0.13028265535831451], "cos_innovation_negg": [0.7281479239463806, 0.12392101436853409, 0.09547808766365051, 0.09054690599441528, 0.12905195355415344, 0.14604750275611877, 0.15337306261062622, 0.12187179177999496, 0.09774170815944672, 0.13584667444229126, 0.21536147594451904, 0.1164068728685379, 0.12027052044868469, 0.10585986077785492, 0.1531606912612915, 0.07483823597431183, 0.11418572813272476, 0.1420007050037384, 0.11972622573375702, 0.1401602178812027, 0.15721440315246582, 0.15242041647434235, 0.1132228672504425, 0.12021379917860031, 0.11057637631893158, 0.15405189990997314, 0.14090479910373688, 0.06371747702360153, 0.059767965227365494, 0.18710459768772125, 0.10280166566371918, 0.07482711970806122, 0.08285433799028397, 0.18478065729141235, 0.08931631594896317, 0.12666559219360352, 0.14000849425792694, 0.019589923322200775, 0.16820503771305084, 0.13894467055797577, 0.0829760879278183, 0.07348090410232544, 0.13687270879745483, 0.06221502274274826, 0.12125816196203232, 0.135206937789917, 0.08886708319187164, 0.07825340330600739, 0.22789394855499268, 0.14402498304843903, 0.17377740144729614, 0.1298290193080902, 0.14159110188484192, 0.14188897609710693, 0.1107911467552185, 0.07106991857290268, 0.10498370230197906, 0.10174622386693954, 0.14011073112487793, 0.13028265535831451], "cos_apical_negg": [0.7281479239463806, 0.12392101436853409, 0.09547808766365051, 0.09054690599441528, 0.12905195355415344, 0.14604750275611877, 0.15337306261062622, 0.12187179177999496, 0.09774170815944672, 0.13584667444229126, 0.21536147594451904, 0.1164068728685379, 0.12027052044868469, 0.10585986077785492, 0.1531606912612915, 0.07483823597431183, 0.11418572813272476, 0.1420007050037384, 0.11972622573375702, 0.1401602178812027, 0.15721440315246582, 0.15242041647434235, 0.1132228672504425, 0.12021379917860031, 0.11057637631893158, 0.15405189990997314, 0.14090479910373688, 0.06371747702360153, 0.059767965227365494, 0.18710459768772125, 0.10280166566371918, 0.07482711970806122, 0.08285433799028397, 0.18478065729141235, 0.08931631594896317, 0.12666559219360352, 0.14000849425792694, 0.019589923322200775, 0.16820503771305084, 0.13894467055797577, 0.0829760879278183, 0.07348090410232544, 0.13687270879745483, 0.06221502274274826, 0.12125816196203232, 0.135206937789917, 0.08886708319187164, 0.07825340330600739, 0.22789394855499268, 0.14402498304843903, 0.17377740144729614, 0.1298290193080902, 0.14159110188484192, 0.14188897609710693, 0.1107911467552185, 0.07106991857290268, 0.10498370230197906, 0.10174622386693954, 0.14011073112487793, 0.13028265535831451], "cos_Ac_negg": [0.7281479239463806, 0.12392101436853409, 0.09547808766365051, 0.09054690599441528, 0.12905195355415344, 0.14604750275611877, 0.15337306261062622, 0.12187179177999496, 0.09774170815944672, 0.13584667444229126, 0.21536147594451904, 0.1164068728685379, 0.12027052044868469, 0.10585986077785492, 0.1531606912612915, 0.07483823597431183, 0.11418572813272476, 0.1420007050037384, 0.11972622573375702, 0.1401602178812027, 0.15721440315246582, 0.15242041647434235, 0.1132228672504425, 0.12021379917860031, 0.11057637631893158, 0.15405189990997314, 0.14090479910373688, 0.06371747702360153, 0.059767965227365494, 0.18710459768772125, 0.10280166566371918, 0.07482711970806122, 0.08285433799028397, 0.18478065729141235, 0.08931631594896317, 0.12666559219360352, 0.14000849425792694, 0.019589923322200775, 0.16820503771305084, 0.13894467055797577, 0.0829760879278183, 0.07348090410232544, 0.13687270879745483, 0.06221502274274826, 0.12125816196203232, 0.135206937789917, 0.08886708319187164, 0.07825340330600739, 0.22789394855499268, 0.14402498304843903, 0.17377740144729614, 0.1298290193080902, 0.14159110188484192, 0.14188897609710693, 0.1107911467552185, 0.07106991857290268, 0.10498370230197906, 0.10174622386693954, 0.14011073112487793, 0.13028265535831451], "r_norm": [1.5385098457336426, 1.3946441411972046, 1.3732115030288696, 1.473698377609253, 1.3236061334609985, 1.459417700767517, 1.4585952758789062, 1.3510830402374268, 1.446205496788025, 1.4405953884124756, 1.402955174446106, 1.4176883697509766, 1.4337135553359985, 1.3812685012817383, 1.4277366399765015, 1.352005958557129, 1.4280941486358643, 1.4436635971069336, 1.4379782676696777, 1.5051898956298828, 1.4516010284423828, 1.4467995166778564, 1.4059423208236694, 1.4329462051391602, 1.3901646137237549, 1.382716417312622, 1.4080452919006348, 1.3788657188415527, 1.4257616996765137, 1.435696005821228, 1.3665878772735596, 1.4647650718688965, 1.4434199333190918, 1.3834577798843384, 1.472854495048523, 1.3528417348861694, 1.3988453149795532, 1.3553028106689453, 1.5008589029312134, 1.300635576248169, 1.430314064025879, 1.522364616394043, 1.430729627609253, 1.40725839138031, 1.4841355085372925, 1.3962287902832031, 1.384103536605835, 1.3545489311218262, 1.4415491819381714, 1.4015021324157715, 1.419175624847412, 1.460895299911499, 1.4078277349472046, 1.42849600315094, 1.3868865966796875, 1.3715262413024902, 1.4998993873596191, 1.386703610420227, 1.4405872821807861, 1.491916537284851]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3961, "test_loss": 1.7338738037109376}, {"step": 1500, "epoch": 3, "train_loss": 1.6361267566680908, "cos_r_negg": [0.7734960317611694, 0.18210665881633759, 0.2143729031085968, 0.16696524620056152, 0.223366379737854, 0.23097896575927734, 0.24401769042015076, 0.20935609936714172, 0.2151353359222412, 0.24124287068843842, 0.26522234082221985, 0.1791776418685913, 0.18872801959514618, 0.22723087668418884, 0.22877724468708038, 0.19398155808448792, 0.18778124451637268, 0.23628415167331696, 0.1962682157754898, 0.22019341588020325, 0.2260873019695282, 0.21276649832725525, 0.20922383666038513, 0.19304409623146057, 0.18577854335308075, 0.23666685819625854, 0.2094118893146515, 0.1702786087989807, 0.12817372381687164, 0.2779242992401123, 0.21197038888931274, 0.1859072595834732, 0.15039877593517303, 0.2078823447227478, 0.2020535171031952, 0.20461326837539673, 0.21361057460308075, 0.1343567818403244, 0.23304277658462524, 0.21160243451595306, 0.17120306193828583, 0.18632130324840546, 0.22008027136325836, 0.1365632712841034, 0.18422389030456543, 0.19866733253002167, 0.18516062200069427, 0.18008321523666382, 0.2631976902484894, 0.24667522311210632, 0.24928739666938782, 0.23282751441001892, 0.225050687789917, 0.19867534935474396, 0.17840945720672607, 0.14314600825309753, 0.20365238189697266, 0.17782405018806458, 0.231987863779068, 0.207154780626297], "cos_innovation_negg": [0.7734960317611694, 0.18210665881633759, 0.2143729031085968, 0.16696524620056152, 0.223366379737854, 0.23097896575927734, 0.24401769042015076, 0.20935609936714172, 0.2151353359222412, 0.24124287068843842, 0.26522234082221985, 0.1791776418685913, 0.18872801959514618, 0.22723087668418884, 0.22877724468708038, 0.19398155808448792, 0.18778124451637268, 0.23628415167331696, 0.1962682157754898, 0.22019341588020325, 0.2260873019695282, 0.21276649832725525, 0.20922383666038513, 0.19304409623146057, 0.18577854335308075, 0.23666685819625854, 0.2094118893146515, 0.1702786087989807, 0.12817372381687164, 0.2779242992401123, 0.21197038888931274, 0.1859072595834732, 0.15039877593517303, 0.2078823447227478, 0.2020535171031952, 0.20461326837539673, 0.21361057460308075, 0.1343567818403244, 0.23304277658462524, 0.21160243451595306, 0.17120306193828583, 0.18632130324840546, 0.22008027136325836, 0.1365632712841034, 0.18422389030456543, 0.19866733253002167, 0.18516062200069427, 0.18008321523666382, 0.2631976902484894, 0.24667522311210632, 0.24928739666938782, 0.23282751441001892, 0.225050687789917, 0.19867534935474396, 0.17840945720672607, 0.14314600825309753, 0.20365238189697266, 0.17782405018806458, 0.231987863779068, 0.207154780626297], "cos_apical_negg": [0.7734960317611694, 0.18210665881633759, 0.2143729031085968, 0.16696524620056152, 0.223366379737854, 0.23097896575927734, 0.24401769042015076, 0.20935609936714172, 0.2151353359222412, 0.24124287068843842, 0.26522234082221985, 0.1791776418685913, 0.18872801959514618, 0.22723087668418884, 0.22877724468708038, 0.19398155808448792, 0.18778124451637268, 0.23628415167331696, 0.1962682157754898, 0.22019341588020325, 0.2260873019695282, 0.21276649832725525, 0.20922383666038513, 0.19304409623146057, 0.18577854335308075, 0.23666685819625854, 0.2094118893146515, 0.1702786087989807, 0.12817372381687164, 0.2779242992401123, 0.21197038888931274, 0.1859072595834732, 0.15039877593517303, 0.2078823447227478, 0.2020535171031952, 0.20461326837539673, 0.21361057460308075, 0.1343567818403244, 0.23304277658462524, 0.21160243451595306, 0.17120306193828583, 0.18632130324840546, 0.22008027136325836, 0.1365632712841034, 0.18422389030456543, 0.19866733253002167, 0.18516062200069427, 0.18008321523666382, 0.2631976902484894, 0.24667522311210632, 0.24928739666938782, 0.23282751441001892, 0.225050687789917, 0.19867534935474396, 0.17840945720672607, 0.14314600825309753, 0.20365238189697266, 0.17782405018806458, 0.231987863779068, 0.207154780626297], "cos_Ac_negg": [0.7734960317611694, 0.18210665881633759, 0.2143729031085968, 0.16696524620056152, 0.223366379737854, 0.23097896575927734, 0.24401769042015076, 0.20935609936714172, 0.2151353359222412, 0.24124287068843842, 0.26522234082221985, 0.1791776418685913, 0.18872801959514618, 0.22723087668418884, 0.22877724468708038, 0.19398155808448792, 0.18778124451637268, 0.23628415167331696, 0.1962682157754898, 0.22019341588020325, 0.2260873019695282, 0.21276649832725525, 0.20922383666038513, 0.19304409623146057, 0.18577854335308075, 0.23666685819625854, 0.2094118893146515, 0.1702786087989807, 0.12817372381687164, 0.2779242992401123, 0.21197038888931274, 0.1859072595834732, 0.15039877593517303, 0.2078823447227478, 0.2020535171031952, 0.20461326837539673, 0.21361057460308075, 0.1343567818403244, 0.23304277658462524, 0.21160243451595306, 0.17120306193828583, 0.18632130324840546, 0.22008027136325836, 0.1365632712841034, 0.18422389030456543, 0.19866733253002167, 0.18516062200069427, 0.18008321523666382, 0.2631976902484894, 0.24667522311210632, 0.24928739666938782, 0.23282751441001892, 0.225050687789917, 0.19867534935474396, 0.17840945720672607, 0.14314600825309753, 0.20365238189697266, 0.17782405018806458, 0.231987863779068, 0.207154780626297], "r_norm": [1.3281257152557373, 1.1315699815750122, 1.1213579177856445, 1.1901010274887085, 1.0870311260223389, 1.1807039976119995, 1.178901195526123, 1.0945137739181519, 1.1682536602020264, 1.173628568649292, 1.142259120941162, 1.1437677145004272, 1.1591336727142334, 1.1198049783706665, 1.1658374071121216, 1.0912506580352783, 1.1632111072540283, 1.173578143119812, 1.180147409439087, 1.2163411378860474, 1.1832292079925537, 1.1813993453979492, 1.142048954963684, 1.160042405128479, 1.1288676261901855, 1.1226162910461426, 1.1491090059280396, 1.1143860816955566, 1.151631474494934, 1.172351360321045, 1.112385869026184, 1.1777746677398682, 1.1669807434082031, 1.1205697059631348, 1.1949219703674316, 1.0972596406936646, 1.1358901262283325, 1.0865752696990967, 1.22475004196167, 1.060443639755249, 1.158452033996582, 1.230194330215454, 1.1615619659423828, 1.1306216716766357, 1.1975423097610474, 1.128929615020752, 1.1192066669464111, 1.0935320854187012, 1.1769263744354248, 1.1414422988891602, 1.1587833166122437, 1.1971298456192017, 1.1401057243347168, 1.1485185623168945, 1.1235904693603516, 1.106951355934143, 1.2120249271392822, 1.1215543746948242, 1.1716290712356567, 1.2104387283325195]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4034, "test_loss": 1.7163406982421876}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4081, "test_loss": 1.7000597778320312}], "final": {"test_acc": 0.4081, "test_loss": 1.7000597778320312, "wall_s": 89.89227414131165, "cos_r_negg": [0.7752143740653992, 0.31061097979545593, 0.284453809261322, 0.2741464376449585, 0.32614728808403015, 0.3158882260322571, 0.33913978934288025, 0.31656625866889954, 0.28921735286712646, 0.35459041595458984, 0.3732917904853821, 0.28232884407043457, 0.30247026681900024, 0.32301804423332214, 0.30722132325172424, 0.28768283128738403, 0.2930471897125244, 0.3490557074546814, 0.29538583755493164, 0.30906277894973755, 0.3300880789756775, 0.3140241801738739, 0.30195850133895874, 0.2835580110549927, 0.27018964290618896, 0.32356083393096924, 0.31137341260910034, 0.26487112045288086, 0.22629129886627197, 0.3581220507621765, 0.31310611963272095, 0.26235225796699524, 0.27234265208244324, 0.32337042689323425, 0.30165669322013855, 0.3499797582626343, 0.31107616424560547, 0.25617921352386475, 0.34797993302345276, 0.3340298533439636, 0.275849312543869, 0.25278186798095703, 0.3374643623828888, 0.26264965534210205, 0.2953434884548187, 0.2732592225074768, 0.28225594758987427, 0.2908916473388672, 0.3798222541809082, 0.3421385884284973, 0.36819544434547424, 0.3378247022628784, 0.32240355014801025, 0.31067517399787903, 0.29638367891311646, 0.2599790394306183, 0.28136196732521057, 0.2625473737716675, 0.3199317455291748, 0.3018702268600464], "cos_innovation_negg": [0.7752143740653992, 0.31061097979545593, 0.284453809261322, 0.2741464376449585, 0.32614728808403015, 0.3158882260322571, 0.33913978934288025, 0.31656625866889954, 0.28921735286712646, 0.35459041595458984, 0.3732917904853821, 0.28232884407043457, 0.30247026681900024, 0.32301804423332214, 0.30722132325172424, 0.28768283128738403, 0.2930471897125244, 0.3490557074546814, 0.29538583755493164, 0.30906277894973755, 0.3300880789756775, 0.3140241801738739, 0.30195850133895874, 0.2835580110549927, 0.27018964290618896, 0.32356083393096924, 0.31137341260910034, 0.26487112045288086, 0.22629129886627197, 0.3581220507621765, 0.31310611963272095, 0.26235225796699524, 0.27234265208244324, 0.32337042689323425, 0.30165669322013855, 0.3499797582626343, 0.31107616424560547, 0.25617921352386475, 0.34797993302345276, 0.3340298533439636, 0.275849312543869, 0.25278186798095703, 0.3374643623828888, 0.26264965534210205, 0.2953434884548187, 0.2732592225074768, 0.28225594758987427, 0.2908916473388672, 0.3798222541809082, 0.3421385884284973, 0.36819544434547424, 0.3378247022628784, 0.32240355014801025, 0.31067517399787903, 0.29638367891311646, 0.2599790394306183, 0.28136196732521057, 0.2625473737716675, 0.3199317455291748, 0.3018702268600464], "cos_apical_negg": [0.7752143740653992, 0.31061097979545593, 0.284453809261322, 0.2741464376449585, 0.32614728808403015, 0.3158882260322571, 0.33913978934288025, 0.31656625866889954, 0.28921735286712646, 0.35459041595458984, 0.3732917904853821, 0.28232884407043457, 0.30247026681900024, 0.32301804423332214, 0.30722132325172424, 0.28768283128738403, 0.2930471897125244, 0.3490557074546814, 0.29538583755493164, 0.30906277894973755, 0.3300880789756775, 0.3140241801738739, 0.30195850133895874, 0.2835580110549927, 0.27018964290618896, 0.32356083393096924, 0.31137341260910034, 0.26487112045288086, 0.22629129886627197, 0.3581220507621765, 0.31310611963272095, 0.26235225796699524, 0.27234265208244324, 0.32337042689323425, 0.30165669322013855, 0.3499797582626343, 0.31107616424560547, 0.25617921352386475, 0.34797993302345276, 0.3340298533439636, 0.275849312543869, 0.25278186798095703, 0.3374643623828888, 0.26264965534210205, 0.2953434884548187, 0.2732592225074768, 0.28225594758987427, 0.2908916473388672, 0.3798222541809082, 0.3421385884284973, 0.36819544434547424, 0.3378247022628784, 0.32240355014801025, 0.31067517399787903, 0.29638367891311646, 0.2599790394306183, 0.28136196732521057, 0.2625473737716675, 0.3199317455291748, 0.3018702268600464], "cos_Ac_negg": [0.7752143740653992, 0.31061097979545593, 0.284453809261322, 0.2741464376449585, 0.32614728808403015, 0.3158882260322571, 0.33913978934288025, 0.31656625866889954, 0.28921735286712646, 0.35459041595458984, 0.3732917904853821, 0.28232884407043457, 0.30247026681900024, 0.32301804423332214, 0.30722132325172424, 0.28768283128738403, 0.2930471897125244, 0.3490557074546814, 0.29538583755493164, 0.30906277894973755, 0.3300880789756775, 0.3140241801738739, 0.30195850133895874, 0.2835580110549927, 0.27018964290618896, 0.32356083393096924, 0.31137341260910034, 0.26487112045288086, 0.22629129886627197, 0.3581220507621765, 0.31310611963272095, 0.26235225796699524, 0.27234265208244324, 0.32337042689323425, 0.30165669322013855, 0.3499797582626343, 0.31107616424560547, 0.25617921352386475, 0.34797993302345276, 0.3340298533439636, 0.275849312543869, 0.25278186798095703, 0.3374643623828888, 0.26264965534210205, 0.2953434884548187, 0.2732592225074768, 0.28225594758987427, 0.2908916473388672, 0.3798222541809082, 0.3421385884284973, 0.36819544434547424, 0.3378247022628784, 0.32240355014801025, 0.31067517399787903, 0.29638367891311646, 0.2599790394306183, 0.28136196732521057, 0.2625473737716675, 0.3199317455291748, 0.3018702268600464]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d60_s2.json b/results/scale_v3_cifar10_sdil_w64_d60_s2.json new file mode 100644 index 0000000..a3772d0 --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d60_s2.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 2, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d60_s2", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.456728219985962, "cos_r_negg": [0.08611082285642624, 0.017612537369132042, -0.001690153731033206, -0.08085684478282928, -0.0015296442434191704, 0.03165651485323906, -0.027990909293293953, -0.029639337211847305, -0.046965524554252625, 0.03454640135169029, 0.07331723719835281, 0.00037553790025413036, 0.06050413101911545, 0.0030032265931367874, 0.0030294875614345074, -0.06769862025976181, 0.01334536261856556, 0.03393281251192093, -0.039688341319561005, 0.017278827726840973, -0.016950123012065887, 0.04365086928009987, -0.039775408804416656, -0.043039705604314804, 0.0014046858996152878, 0.001827881089411676, -0.0622515045106411, 0.010696405544877052, -0.0036294935271143913, -0.05205393210053444, 0.061973027884960175, 0.04007808864116669, 0.024303218349814415, -0.059550732374191284, -0.06498114764690399, -0.02916010282933712, -0.038352109491825104, -0.040698617696762085, 0.03185011446475983, -0.007801948115229607, -0.028400635346770287, 0.032049812376499176, 0.022648900747299194, -0.024968545883893967, -0.04406441003084183, 0.04823513701558113, 0.03871341049671173, -0.003967022988945246, 0.09255938977003098, 0.12419933080673218, 0.05303552746772766, 0.017109673470258713, 0.07447409629821777, -0.030246540904045105, -0.03758876770734787, -0.09144528210163116, -0.09526985883712769, 0.05922526866197586, 0.012512996792793274, 0.015913574025034904], "cos_innovation_negg": [0.08611082285642624, 0.017612537369132042, -0.001690153731033206, -0.08085684478282928, -0.0015296442434191704, 0.03165651485323906, -0.027990909293293953, -0.029639337211847305, -0.046965524554252625, 0.03454640135169029, 0.07331723719835281, 0.00037553790025413036, 0.06050413101911545, 0.0030032265931367874, 0.0030294875614345074, -0.06769862025976181, 0.01334536261856556, 0.03393281251192093, -0.039688341319561005, 0.017278827726840973, -0.016950123012065887, 0.04365086928009987, -0.039775408804416656, -0.043039705604314804, 0.0014046858996152878, 0.001827881089411676, -0.0622515045106411, 0.010696405544877052, -0.0036294935271143913, -0.05205393210053444, 0.061973027884960175, 0.04007808864116669, 0.024303218349814415, -0.059550732374191284, -0.06498114764690399, -0.02916010282933712, -0.038352109491825104, -0.040698617696762085, 0.03185011446475983, -0.007801948115229607, -0.028400635346770287, 0.032049812376499176, 0.022648900747299194, -0.024968545883893967, -0.04406441003084183, 0.04823513701558113, 0.03871341049671173, -0.003967022988945246, 0.09255938977003098, 0.12419933080673218, 0.05303552746772766, 0.017109673470258713, 0.07447409629821777, -0.030246540904045105, -0.03758876770734787, -0.09144528210163116, -0.09526985883712769, 0.05922526866197586, 0.012512996792793274, 0.015913574025034904], "cos_apical_negg": [0.08611082285642624, 0.017612537369132042, -0.001690153731033206, -0.08085684478282928, -0.0015296442434191704, 0.03165651485323906, -0.027990909293293953, -0.029639337211847305, -0.046965524554252625, 0.03454640135169029, 0.07331723719835281, 0.00037553790025413036, 0.06050413101911545, 0.0030032265931367874, 0.0030294875614345074, -0.06769862025976181, 0.01334536261856556, 0.03393281251192093, -0.039688341319561005, 0.017278827726840973, -0.016950123012065887, 0.04365086928009987, -0.039775408804416656, -0.043039705604314804, 0.0014046858996152878, 0.001827881089411676, -0.0622515045106411, 0.010696405544877052, -0.0036294935271143913, -0.05205393210053444, 0.061973027884960175, 0.04007808864116669, 0.024303218349814415, -0.059550732374191284, -0.06498114764690399, -0.02916010282933712, -0.038352109491825104, -0.040698617696762085, 0.03185011446475983, -0.007801948115229607, -0.028400635346770287, 0.032049812376499176, 0.022648900747299194, -0.024968545883893967, -0.04406441003084183, 0.04823513701558113, 0.03871341049671173, -0.003967022988945246, 0.09255938977003098, 0.12419933080673218, 0.05303552746772766, 0.017109673470258713, 0.07447409629821777, -0.030246540904045105, -0.03758876770734787, -0.09144528210163116, -0.09526985883712769, 0.05922526866197586, 0.012512996792793274, 0.015913574025034904], "cos_Ac_negg": [0.08611082285642624, 0.017612537369132042, -0.001690153731033206, -0.08085684478282928, -0.0015296442434191704, 0.03165651485323906, -0.027990909293293953, -0.029639337211847305, -0.046965524554252625, 0.03454640135169029, 0.07331723719835281, 0.00037553790025413036, 0.06050413101911545, 0.0030032265931367874, 0.0030294875614345074, -0.06769862025976181, 0.01334536261856556, 0.03393281251192093, -0.039688341319561005, 0.017278827726840973, -0.016950123012065887, 0.04365086928009987, -0.039775408804416656, -0.043039705604314804, 0.0014046858996152878, 0.001827881089411676, -0.0622515045106411, 0.010696405544877052, -0.0036294935271143913, -0.05205393210053444, 0.061973027884960175, 0.04007808864116669, 0.024303218349814415, -0.059550732374191284, -0.06498114764690399, -0.02916010282933712, -0.038352109491825104, -0.040698617696762085, 0.03185011446475983, -0.007801948115229607, -0.028400635346770287, 0.032049812376499176, 0.022648900747299194, -0.024968545883893967, -0.04406441003084183, 0.04823513701558113, 0.03871341049671173, -0.003967022988945246, 0.09255938977003098, 0.12419933080673218, 0.05303552746772766, 0.017109673470258713, 0.07447409629821777, -0.030246540904045105, -0.03758876770734787, -0.09144528210163116, -0.09526985883712769, 0.05922526866197586, 0.012512996792793274, 0.015913574025034904], "r_norm": [2.5279881954193115, 2.3028414249420166, 2.5141243934631348, 2.456221103668213, 2.423701286315918, 2.3154499530792236, 2.501948356628418, 2.330960750579834, 2.4371628761291504, 2.366441011428833, 2.285327434539795, 2.3079309463500977, 2.4701573848724365, 2.4401168823242188, 2.4964599609375, 2.441318988800049, 2.582400321960449, 2.2878623008728027, 2.432720184326172, 2.3879125118255615, 2.463024377822876, 2.457271099090576, 2.340888500213623, 2.390347957611084, 2.425302743911743, 2.503244400024414, 2.5851855278015137, 2.472188949584961, 2.3574464321136475, 2.442430019378662, 2.4181876182556152, 2.347317695617676, 2.410494565963745, 2.5535507202148438, 2.496727466583252, 2.357452869415283, 2.324160099029541, 2.335726737976074, 2.4148497581481934, 2.3808083534240723, 2.4768099784851074, 2.4362759590148926, 2.3489975929260254, 2.473440647125244, 2.449836254119873, 2.4900002479553223, 2.3947534561157227, 2.3510684967041016, 2.558475971221924, 2.4334030151367188, 2.3637609481811523, 2.3625833988189697, 2.422903537750244, 2.4801979064941406, 2.429744243621826, 2.346132278442383, 2.2917542457580566, 2.5047624111175537, 2.3685553073883057, 2.3860812187194824], "ldr": {"dL_sdil": -0.08301210403442383, "dL_bp": -0.36100101470947266, "ratio": 0.2299497803385138}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3623, "test_loss": 1.8130749633789063}, {"step": 500, "epoch": 1, "train_loss": 1.647667646408081, "cos_r_negg": [0.641769528388977, 0.04386383295059204, 0.07463890314102173, 0.021204303950071335, 0.057012878358364105, 0.07797746360301971, 0.04838663712143898, 0.10113523155450821, 0.041023753583431244, 0.09492979943752289, 0.11917727440595627, -0.017427703365683556, 0.04937848076224327, 0.0198967307806015, 0.08508764952421188, 0.07521022856235504, 0.06729884445667267, 0.12150822579860687, 0.09932173788547516, 0.137334406375885, 0.0035226233303546906, 0.06921856850385666, 0.09367518126964569, -0.041014622896909714, 0.09224380552768707, 0.018353065475821495, 0.02426125481724739, 0.047729190438985825, 0.057038288563489914, 0.0903923362493515, 0.08484892547130585, 0.12854154407978058, 0.06663370132446289, 0.03489287570118904, 0.10823875665664673, 0.013422808609902859, 0.00047389522660523653, 0.09022317826747894, 0.027861082926392555, 0.1001090258359909, 0.03989126533269882, 0.08583913743495941, 0.03402452915906906, 0.058253757655620575, 0.010880635119974613, 0.049959130585193634, 0.04951344430446625, -0.008289683610200882, 0.03429049625992775, 0.06877272576093674, 0.09401653707027435, 0.006343696266412735, 0.06847921013832092, 0.07727882266044617, 0.03374047577381134, 0.017143288627266884, 0.0008720592595636845, 0.12311574816703796, 0.01842271164059639, 0.03322608023881912], "cos_innovation_negg": [0.641769528388977, 0.04386383295059204, 0.07463890314102173, 0.021204303950071335, 0.057012878358364105, 0.07797746360301971, 0.04838663712143898, 0.10113523155450821, 0.041023753583431244, 0.09492979943752289, 0.11917727440595627, -0.017427703365683556, 0.04937848076224327, 0.0198967307806015, 0.08508764952421188, 0.07521022856235504, 0.06729884445667267, 0.12150822579860687, 0.09932173788547516, 0.137334406375885, 0.0035226233303546906, 0.06921856850385666, 0.09367518126964569, -0.041014622896909714, 0.09224380552768707, 0.018353065475821495, 0.02426125481724739, 0.047729190438985825, 0.057038288563489914, 0.0903923362493515, 0.08484892547130585, 0.12854154407978058, 0.06663370132446289, 0.03489287570118904, 0.10823875665664673, 0.013422808609902859, 0.00047389522660523653, 0.09022317826747894, 0.027861082926392555, 0.1001090258359909, 0.03989126533269882, 0.08583913743495941, 0.03402452915906906, 0.058253757655620575, 0.010880635119974613, 0.049959130585193634, 0.04951344430446625, -0.008289683610200882, 0.03429049625992775, 0.06877272576093674, 0.09401653707027435, 0.006343696266412735, 0.06847921013832092, 0.07727882266044617, 0.03374047577381134, 0.017143288627266884, 0.0008720592595636845, 0.12311574816703796, 0.01842271164059639, 0.03322608023881912], "cos_apical_negg": [0.641769528388977, 0.04386383295059204, 0.07463890314102173, 0.021204303950071335, 0.057012878358364105, 0.07797746360301971, 0.04838663712143898, 0.10113523155450821, 0.041023753583431244, 0.09492979943752289, 0.11917727440595627, -0.017427703365683556, 0.04937848076224327, 0.0198967307806015, 0.08508764952421188, 0.07521022856235504, 0.06729884445667267, 0.12150822579860687, 0.09932173788547516, 0.137334406375885, 0.0035226233303546906, 0.06921856850385666, 0.09367518126964569, -0.041014622896909714, 0.09224380552768707, 0.018353065475821495, 0.02426125481724739, 0.047729190438985825, 0.057038288563489914, 0.0903923362493515, 0.08484892547130585, 0.12854154407978058, 0.06663370132446289, 0.03489287570118904, 0.10823875665664673, 0.013422808609902859, 0.00047389522660523653, 0.09022317826747894, 0.027861082926392555, 0.1001090258359909, 0.03989126533269882, 0.08583913743495941, 0.03402452915906906, 0.058253757655620575, 0.010880635119974613, 0.049959130585193634, 0.04951344430446625, -0.008289683610200882, 0.03429049625992775, 0.06877272576093674, 0.09401653707027435, 0.006343696266412735, 0.06847921013832092, 0.07727882266044617, 0.03374047577381134, 0.017143288627266884, 0.0008720592595636845, 0.12311574816703796, 0.01842271164059639, 0.03322608023881912], "cos_Ac_negg": [0.641769528388977, 0.04386383295059204, 0.07463890314102173, 0.021204303950071335, 0.057012878358364105, 0.07797746360301971, 0.04838663712143898, 0.10113523155450821, 0.041023753583431244, 0.09492979943752289, 0.11917727440595627, -0.017427703365683556, 0.04937848076224327, 0.0198967307806015, 0.08508764952421188, 0.07521022856235504, 0.06729884445667267, 0.12150822579860687, 0.09932173788547516, 0.137334406375885, 0.0035226233303546906, 0.06921856850385666, 0.09367518126964569, -0.041014622896909714, 0.09224380552768707, 0.018353065475821495, 0.02426125481724739, 0.047729190438985825, 0.057038288563489914, 0.0903923362493515, 0.08484892547130585, 0.12854154407978058, 0.06663370132446289, 0.03489287570118904, 0.10823875665664673, 0.013422808609902859, 0.00047389522660523653, 0.09022317826747894, 0.027861082926392555, 0.1001090258359909, 0.03989126533269882, 0.08583913743495941, 0.03402452915906906, 0.058253757655620575, 0.010880635119974613, 0.049959130585193634, 0.04951344430446625, -0.008289683610200882, 0.03429049625992775, 0.06877272576093674, 0.09401653707027435, 0.006343696266412735, 0.06847921013832092, 0.07727882266044617, 0.03374047577381134, 0.017143288627266884, 0.0008720592595636845, 0.12311574816703796, 0.01842271164059639, 0.03322608023881912], "r_norm": [1.8209681510925293, 1.6141014099121094, 1.770363688468933, 1.7218115329742432, 1.7118332386016846, 1.6286389827728271, 1.7343182563781738, 1.622497797012329, 1.6917554140090942, 1.664224624633789, 1.6012752056121826, 1.5958118438720703, 1.7321736812591553, 1.7117646932601929, 1.752622127532959, 1.7185804843902588, 1.8102052211761475, 1.6078778505325317, 1.7049047946929932, 1.6692615747451782, 1.6977629661560059, 1.7430680990219116, 1.6420073509216309, 1.6566094160079956, 1.713047981262207, 1.7564054727554321, 1.7947967052459717, 1.7278015613555908, 1.6431357860565186, 1.7166568040847778, 1.6846169233322144, 1.6454222202301025, 1.6968512535095215, 1.8022133111953735, 1.7575297355651855, 1.6467690467834473, 1.589847207069397, 1.6284513473510742, 1.6974207162857056, 1.6721841096878052, 1.7269549369812012, 1.7283450365066528, 1.646378517150879, 1.7125473022460938, 1.6987946033477783, 1.7626466751098633, 1.681551218032837, 1.6376783847808838, 1.788825273513794, 1.690885305404663, 1.6665555238723755, 1.6487572193145752, 1.704043984413147, 1.7634512186050415, 1.6988167762756348, 1.6549032926559448, 1.6100205183029175, 1.7549748420715332, 1.6675605773925781, 1.6841533184051514]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3973, "test_loss": 1.7619666381835937}, {"step": 1000, "epoch": 2, "train_loss": 1.8276193141937256, "cos_r_negg": [0.7082642912864685, 0.15606874227523804, 0.13993577659130096, 0.08397994190454483, 0.13277322053909302, 0.18050548434257507, 0.099637471139431, 0.17577674984931946, 0.10105134546756744, 0.163823202252388, 0.18464042246341705, 0.08680509030818939, 0.15214112401008606, 0.11036652326583862, 0.17029833793640137, 0.14198023080825806, 0.1445707380771637, 0.17575785517692566, 0.1562148630619049, 0.1746155321598053, 0.0995960533618927, 0.1480758935213089, 0.14604145288467407, 0.02808878943324089, 0.19037765264511108, 0.045590486377477646, 0.08480807393789291, 0.11635389924049377, 0.13613396883010864, 0.16870668530464172, 0.21953356266021729, 0.16809096932411194, 0.13865220546722412, 0.1713135838508606, 0.13746324181556702, 0.06107468530535698, 0.06018224358558655, 0.15696823596954346, 0.07231871783733368, 0.11910079419612885, 0.1783832162618637, 0.1945352554321289, 0.15713657438755035, 0.13438205420970917, 0.08120027929544449, 0.1218709722161293, 0.1404557228088379, 0.06876237690448761, 0.11056104302406311, 0.19422562420368195, 0.12154794484376907, 0.0541163831949234, 0.11579012870788574, 0.1515892893075943, 0.0972316563129425, 0.06939049065113068, 0.08539848029613495, 0.15680289268493652, 0.11893978714942932, 0.07244283705949783], "cos_innovation_negg": [0.7082642912864685, 0.15606874227523804, 0.13993577659130096, 0.08397994190454483, 0.13277322053909302, 0.18050548434257507, 0.099637471139431, 0.17577674984931946, 0.10105134546756744, 0.163823202252388, 0.18464042246341705, 0.08680509030818939, 0.15214112401008606, 0.11036652326583862, 0.17029833793640137, 0.14198023080825806, 0.1445707380771637, 0.17575785517692566, 0.1562148630619049, 0.1746155321598053, 0.0995960533618927, 0.1480758935213089, 0.14604145288467407, 0.02808878943324089, 0.19037765264511108, 0.045590486377477646, 0.08480807393789291, 0.11635389924049377, 0.13613396883010864, 0.16870668530464172, 0.21953356266021729, 0.16809096932411194, 0.13865220546722412, 0.1713135838508606, 0.13746324181556702, 0.06107468530535698, 0.06018224358558655, 0.15696823596954346, 0.07231871783733368, 0.11910079419612885, 0.1783832162618637, 0.1945352554321289, 0.15713657438755035, 0.13438205420970917, 0.08120027929544449, 0.1218709722161293, 0.1404557228088379, 0.06876237690448761, 0.11056104302406311, 0.19422562420368195, 0.12154794484376907, 0.0541163831949234, 0.11579012870788574, 0.1515892893075943, 0.0972316563129425, 0.06939049065113068, 0.08539848029613495, 0.15680289268493652, 0.11893978714942932, 0.07244283705949783], "cos_apical_negg": [0.7082642912864685, 0.15606874227523804, 0.13993577659130096, 0.08397994190454483, 0.13277322053909302, 0.18050548434257507, 0.099637471139431, 0.17577674984931946, 0.10105134546756744, 0.163823202252388, 0.18464042246341705, 0.08680509030818939, 0.15214112401008606, 0.11036652326583862, 0.17029833793640137, 0.14198023080825806, 0.1445707380771637, 0.17575785517692566, 0.1562148630619049, 0.1746155321598053, 0.0995960533618927, 0.1480758935213089, 0.14604145288467407, 0.02808878943324089, 0.19037765264511108, 0.045590486377477646, 0.08480807393789291, 0.11635389924049377, 0.13613396883010864, 0.16870668530464172, 0.21953356266021729, 0.16809096932411194, 0.13865220546722412, 0.1713135838508606, 0.13746324181556702, 0.06107468530535698, 0.06018224358558655, 0.15696823596954346, 0.07231871783733368, 0.11910079419612885, 0.1783832162618637, 0.1945352554321289, 0.15713657438755035, 0.13438205420970917, 0.08120027929544449, 0.1218709722161293, 0.1404557228088379, 0.06876237690448761, 0.11056104302406311, 0.19422562420368195, 0.12154794484376907, 0.0541163831949234, 0.11579012870788574, 0.1515892893075943, 0.0972316563129425, 0.06939049065113068, 0.08539848029613495, 0.15680289268493652, 0.11893978714942932, 0.07244283705949783], "cos_Ac_negg": [0.7082642912864685, 0.15606874227523804, 0.13993577659130096, 0.08397994190454483, 0.13277322053909302, 0.18050548434257507, 0.099637471139431, 0.17577674984931946, 0.10105134546756744, 0.163823202252388, 0.18464042246341705, 0.08680509030818939, 0.15214112401008606, 0.11036652326583862, 0.17029833793640137, 0.14198023080825806, 0.1445707380771637, 0.17575785517692566, 0.1562148630619049, 0.1746155321598053, 0.0995960533618927, 0.1480758935213089, 0.14604145288467407, 0.02808878943324089, 0.19037765264511108, 0.045590486377477646, 0.08480807393789291, 0.11635389924049377, 0.13613396883010864, 0.16870668530464172, 0.21953356266021729, 0.16809096932411194, 0.13865220546722412, 0.1713135838508606, 0.13746324181556702, 0.06107468530535698, 0.06018224358558655, 0.15696823596954346, 0.07231871783733368, 0.11910079419612885, 0.1783832162618637, 0.1945352554321289, 0.15713657438755035, 0.13438205420970917, 0.08120027929544449, 0.1218709722161293, 0.1404557228088379, 0.06876237690448761, 0.11056104302406311, 0.19422562420368195, 0.12154794484376907, 0.0541163831949234, 0.11579012870788574, 0.1515892893075943, 0.0972316563129425, 0.06939049065113068, 0.08539848029613495, 0.15680289268493652, 0.11893978714942932, 0.07244283705949783], "r_norm": [1.560402750968933, 1.321803092956543, 1.443197250366211, 1.3973808288574219, 1.39340341091156, 1.3360519409179688, 1.409365177154541, 1.3380569219589233, 1.3725931644439697, 1.3652260303497314, 1.319079875946045, 1.2982532978057861, 1.4124038219451904, 1.3984441757202148, 1.4320340156555176, 1.4051663875579834, 1.4725010395050049, 1.3174095153808594, 1.3972947597503662, 1.3700189590454102, 1.3876097202301025, 1.42263925075531, 1.3361870050430298, 1.3371045589447021, 1.4042155742645264, 1.418631911277771, 1.4627033472061157, 1.4107178449630737, 1.336469292640686, 1.4001972675323486, 1.3799591064453125, 1.3554964065551758, 1.382567048072815, 1.467519998550415, 1.4245901107788086, 1.3319884538650513, 1.2937085628509521, 1.3294702768325806, 1.3804389238357544, 1.3603155612945557, 1.4023139476776123, 1.4155194759368896, 1.3401696681976318, 1.3975931406021118, 1.3780710697174072, 1.4379241466522217, 1.3687770366668701, 1.3255527019500732, 1.4513036012649536, 1.3843755722045898, 1.3571271896362305, 1.3380626440048218, 1.3901430368423462, 1.4462288618087769, 1.3783987760543823, 1.3366279602050781, 1.3039381504058838, 1.4371603727340698, 1.3566285371780396, 1.3604552745819092]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3977, "test_loss": 1.72474951171875}, {"step": 1500, "epoch": 3, "train_loss": 1.620103359222412, "cos_r_negg": [0.7739584445953369, 0.2080114483833313, 0.18250665068626404, 0.18902087211608887, 0.1994839310646057, 0.28379762172698975, 0.19450971484184265, 0.23946624994277954, 0.19351816177368164, 0.2353816032409668, 0.2686036229133606, 0.16681858897209167, 0.2402220368385315, 0.21861770749092102, 0.26832011342048645, 0.2191285789012909, 0.25580447912216187, 0.2783512771129608, 0.2733460068702698, 0.2626396119594574, 0.18571925163269043, 0.25916624069213867, 0.2586159110069275, 0.1378674954175949, 0.26208797097206116, 0.15676844120025635, 0.2100072205066681, 0.2391136884689331, 0.227627694606781, 0.2663933336734772, 0.28531062602996826, 0.2844960689544678, 0.20642364025115967, 0.25035232305526733, 0.2279733121395111, 0.18928663432598114, 0.18779107928276062, 0.24189972877502441, 0.2109822928905487, 0.18043291568756104, 0.2797839045524597, 0.25708574056625366, 0.21490389108657837, 0.22852283716201782, 0.1815214455127716, 0.281131386756897, 0.22665929794311523, 0.18172207474708557, 0.1800031065940857, 0.2568029761314392, 0.2187817394733429, 0.16893884539604187, 0.22219116985797882, 0.2650783658027649, 0.19234195351600647, 0.223589226603508, 0.2408067137002945, 0.23051846027374268, 0.2247641235589981, 0.1790793240070343], "cos_innovation_negg": [0.7739584445953369, 0.2080114483833313, 0.18250665068626404, 0.18902087211608887, 0.1994839310646057, 0.28379762172698975, 0.19450971484184265, 0.23946624994277954, 0.19351816177368164, 0.2353816032409668, 0.2686036229133606, 0.16681858897209167, 0.2402220368385315, 0.21861770749092102, 0.26832011342048645, 0.2191285789012909, 0.25580447912216187, 0.2783512771129608, 0.2733460068702698, 0.2626396119594574, 0.18571925163269043, 0.25916624069213867, 0.2586159110069275, 0.1378674954175949, 0.26208797097206116, 0.15676844120025635, 0.2100072205066681, 0.2391136884689331, 0.227627694606781, 0.2663933336734772, 0.28531062602996826, 0.2844960689544678, 0.20642364025115967, 0.25035232305526733, 0.2279733121395111, 0.18928663432598114, 0.18779107928276062, 0.24189972877502441, 0.2109822928905487, 0.18043291568756104, 0.2797839045524597, 0.25708574056625366, 0.21490389108657837, 0.22852283716201782, 0.1815214455127716, 0.281131386756897, 0.22665929794311523, 0.18172207474708557, 0.1800031065940857, 0.2568029761314392, 0.2187817394733429, 0.16893884539604187, 0.22219116985797882, 0.2650783658027649, 0.19234195351600647, 0.223589226603508, 0.2408067137002945, 0.23051846027374268, 0.2247641235589981, 0.1790793240070343], "cos_apical_negg": [0.7739584445953369, 0.2080114483833313, 0.18250665068626404, 0.18902087211608887, 0.1994839310646057, 0.28379762172698975, 0.19450971484184265, 0.23946624994277954, 0.19351816177368164, 0.2353816032409668, 0.2686036229133606, 0.16681858897209167, 0.2402220368385315, 0.21861770749092102, 0.26832011342048645, 0.2191285789012909, 0.25580447912216187, 0.2783512771129608, 0.2733460068702698, 0.2626396119594574, 0.18571925163269043, 0.25916624069213867, 0.2586159110069275, 0.1378674954175949, 0.26208797097206116, 0.15676844120025635, 0.2100072205066681, 0.2391136884689331, 0.227627694606781, 0.2663933336734772, 0.28531062602996826, 0.2844960689544678, 0.20642364025115967, 0.25035232305526733, 0.2279733121395111, 0.18928663432598114, 0.18779107928276062, 0.24189972877502441, 0.2109822928905487, 0.18043291568756104, 0.2797839045524597, 0.25708574056625366, 0.21490389108657837, 0.22852283716201782, 0.1815214455127716, 0.281131386756897, 0.22665929794311523, 0.18172207474708557, 0.1800031065940857, 0.2568029761314392, 0.2187817394733429, 0.16893884539604187, 0.22219116985797882, 0.2650783658027649, 0.19234195351600647, 0.223589226603508, 0.2408067137002945, 0.23051846027374268, 0.2247641235589981, 0.1790793240070343], "cos_Ac_negg": [0.7739584445953369, 0.2080114483833313, 0.18250665068626404, 0.18902087211608887, 0.1994839310646057, 0.28379762172698975, 0.19450971484184265, 0.23946624994277954, 0.19351816177368164, 0.2353816032409668, 0.2686036229133606, 0.16681858897209167, 0.2402220368385315, 0.21861770749092102, 0.26832011342048645, 0.2191285789012909, 0.25580447912216187, 0.2783512771129608, 0.2733460068702698, 0.2626396119594574, 0.18571925163269043, 0.25916624069213867, 0.2586159110069275, 0.1378674954175949, 0.26208797097206116, 0.15676844120025635, 0.2100072205066681, 0.2391136884689331, 0.227627694606781, 0.2663933336734772, 0.28531062602996826, 0.2844960689544678, 0.20642364025115967, 0.25035232305526733, 0.2279733121395111, 0.18928663432598114, 0.18779107928276062, 0.24189972877502441, 0.2109822928905487, 0.18043291568756104, 0.2797839045524597, 0.25708574056625366, 0.21490389108657837, 0.22852283716201782, 0.1815214455127716, 0.281131386756897, 0.22665929794311523, 0.18172207474708557, 0.1800031065940857, 0.2568029761314392, 0.2187817394733429, 0.16893884539604187, 0.22219116985797882, 0.2650783658027649, 0.19234195351600647, 0.223589226603508, 0.2408067137002945, 0.23051846027374268, 0.2247641235589981, 0.1790793240070343], "r_norm": [1.3689594268798828, 1.0870769023895264, 1.1857540607452393, 1.1480607986450195, 1.140582799911499, 1.1022882461547852, 1.1591696739196777, 1.1061956882476807, 1.1294915676116943, 1.1219230890274048, 1.0936962366104126, 1.059211015701294, 1.1682244539260864, 1.1568615436553955, 1.1935704946517944, 1.1680881977081299, 1.2146077156066895, 1.0880956649780273, 1.160996437072754, 1.1433653831481934, 1.139925479888916, 1.1733355522155762, 1.1114575862884521, 1.101152777671814, 1.168304681777954, 1.160860300064087, 1.1996692419052124, 1.167079210281372, 1.1029798984527588, 1.164151906967163, 1.150730848312378, 1.1286553144454956, 1.1382898092269897, 1.207364559173584, 1.1758241653442383, 1.0944463014602661, 1.0612165927886963, 1.0988421440124512, 1.1370885372161865, 1.120713472366333, 1.1637184619903564, 1.1658055782318115, 1.1075944900512695, 1.1544240713119507, 1.1362351179122925, 1.1815019845962524, 1.1216727495193481, 1.0837342739105225, 1.1947580575942993, 1.1486496925354004, 1.1149903535842896, 1.0946745872497559, 1.1478476524353027, 1.1931827068328857, 1.132091760635376, 1.0951910018920898, 1.072930097579956, 1.1842303276062012, 1.1149142980575562, 1.1198146343231201]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4013, "test_loss": 1.7225825317382812}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4158, "test_loss": 1.6949622924804688}], "final": {"test_acc": 0.4158, "test_loss": 1.6949622924804688, "wall_s": 117.48857545852661, "cos_r_negg": [0.8220240473747253, 0.317320853471756, 0.31269270181655884, 0.2800901532173157, 0.29436418414115906, 0.3671455979347229, 0.31533852219581604, 0.3758537471294403, 0.2866869270801544, 0.34882116317749023, 0.37997132539749146, 0.2768024206161499, 0.3320702910423279, 0.3260497450828552, 0.33807915449142456, 0.3505033850669861, 0.3352736234664917, 0.3791392743587494, 0.3719269931316376, 0.36095112562179565, 0.29678767919540405, 0.3729771375656128, 0.3468477427959442, 0.2724785804748535, 0.3519517481327057, 0.26703137159347534, 0.29117459058761597, 0.34231114387512207, 0.32633888721466064, 0.3600957989692688, 0.41904565691947937, 0.40246111154556274, 0.3129122853279114, 0.3359273076057434, 0.30861520767211914, 0.3056429624557495, 0.3013440668582916, 0.33988094329833984, 0.3122744560241699, 0.3030822277069092, 0.36874115467071533, 0.35592877864837646, 0.30112260580062866, 0.352790892124176, 0.30509820580482483, 0.36225736141204834, 0.3135794401168823, 0.3113824427127838, 0.3111315071582794, 0.3684026002883911, 0.34208908677101135, 0.2990241050720215, 0.34235721826553345, 0.33697253465652466, 0.31604135036468506, 0.30312058329582214, 0.3260119557380676, 0.3420763611793518, 0.32001587748527527, 0.31079185009002686], "cos_innovation_negg": [0.8220240473747253, 0.317320853471756, 0.31269270181655884, 0.2800901532173157, 0.29436418414115906, 0.3671455979347229, 0.31533852219581604, 0.3758537471294403, 0.2866869270801544, 0.34882116317749023, 0.37997132539749146, 0.2768024206161499, 0.3320702910423279, 0.3260497450828552, 0.33807915449142456, 0.3505033850669861, 0.3352736234664917, 0.3791392743587494, 0.3719269931316376, 0.36095112562179565, 0.29678767919540405, 0.3729771375656128, 0.3468477427959442, 0.2724785804748535, 0.3519517481327057, 0.26703137159347534, 0.29117459058761597, 0.34231114387512207, 0.32633888721466064, 0.3600957989692688, 0.41904565691947937, 0.40246111154556274, 0.3129122853279114, 0.3359273076057434, 0.30861520767211914, 0.3056429624557495, 0.3013440668582916, 0.33988094329833984, 0.3122744560241699, 0.3030822277069092, 0.36874115467071533, 0.35592877864837646, 0.30112260580062866, 0.352790892124176, 0.30509820580482483, 0.36225736141204834, 0.3135794401168823, 0.3113824427127838, 0.3111315071582794, 0.3684026002883911, 0.34208908677101135, 0.2990241050720215, 0.34235721826553345, 0.33697253465652466, 0.31604135036468506, 0.30312058329582214, 0.3260119557380676, 0.3420763611793518, 0.32001587748527527, 0.31079185009002686], "cos_apical_negg": [0.8220240473747253, 0.317320853471756, 0.31269270181655884, 0.2800901532173157, 0.29436418414115906, 0.3671455979347229, 0.31533852219581604, 0.3758537471294403, 0.2866869270801544, 0.34882116317749023, 0.37997132539749146, 0.2768024206161499, 0.3320702910423279, 0.3260497450828552, 0.33807915449142456, 0.3505033850669861, 0.3352736234664917, 0.3791392743587494, 0.3719269931316376, 0.36095112562179565, 0.29678767919540405, 0.3729771375656128, 0.3468477427959442, 0.2724785804748535, 0.3519517481327057, 0.26703137159347534, 0.29117459058761597, 0.34231114387512207, 0.32633888721466064, 0.3600957989692688, 0.41904565691947937, 0.40246111154556274, 0.3129122853279114, 0.3359273076057434, 0.30861520767211914, 0.3056429624557495, 0.3013440668582916, 0.33988094329833984, 0.3122744560241699, 0.3030822277069092, 0.36874115467071533, 0.35592877864837646, 0.30112260580062866, 0.352790892124176, 0.30509820580482483, 0.36225736141204834, 0.3135794401168823, 0.3113824427127838, 0.3111315071582794, 0.3684026002883911, 0.34208908677101135, 0.2990241050720215, 0.34235721826553345, 0.33697253465652466, 0.31604135036468506, 0.30312058329582214, 0.3260119557380676, 0.3420763611793518, 0.32001587748527527, 0.31079185009002686], "cos_Ac_negg": [0.8220240473747253, 0.317320853471756, 0.31269270181655884, 0.2800901532173157, 0.29436418414115906, 0.3671455979347229, 0.31533852219581604, 0.3758537471294403, 0.2866869270801544, 0.34882116317749023, 0.37997132539749146, 0.2768024206161499, 0.3320702910423279, 0.3260497450828552, 0.33807915449142456, 0.3505033850669861, 0.3352736234664917, 0.3791392743587494, 0.3719269931316376, 0.36095112562179565, 0.29678767919540405, 0.3729771375656128, 0.3468477427959442, 0.2724785804748535, 0.3519517481327057, 0.26703137159347534, 0.29117459058761597, 0.34231114387512207, 0.32633888721466064, 0.3600957989692688, 0.41904565691947937, 0.40246111154556274, 0.3129122853279114, 0.3359273076057434, 0.30861520767211914, 0.3056429624557495, 0.3013440668582916, 0.33988094329833984, 0.3122744560241699, 0.3030822277069092, 0.36874115467071533, 0.35592877864837646, 0.30112260580062866, 0.352790892124176, 0.30509820580482483, 0.36225736141204834, 0.3135794401168823, 0.3113824427127838, 0.3111315071582794, 0.3684026002883911, 0.34208908677101135, 0.2990241050720215, 0.34235721826553345, 0.33697253465652466, 0.31604135036468506, 0.30312058329582214, 0.3260119557380676, 0.3420763611793518, 0.32001587748527527, 0.31079185009002686]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d60_s3.json b/results/scale_v3_cifar10_sdil_w64_d60_s3.json new file mode 100644 index 0000000..1700639 --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d60_s3.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 3, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d60_s3", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.416471481323242, "cos_r_negg": [0.003765084780752659, 0.00043217861093580723, 0.0013481127098202705, 0.053083255887031555, 0.012091723270714283, 0.004633747041225433, 0.003966432996094227, -0.026057565584778786, -0.008876107633113861, 0.02871987782418728, 0.04269128292798996, 0.0014793600421398878, 0.025191795080900192, -0.0674225240945816, 0.015505467541515827, -0.06318938732147217, -0.031499456614255905, 0.07636164128780365, 0.023246653378009796, -0.0035817446187138557, 0.04152105748653412, -0.01521938107907772, -0.041440315544605255, 0.038943491876125336, 0.040427740663290024, 0.04755157232284546, 0.04602851718664169, -0.022689521312713623, -0.001713272649794817, -0.015236373990774155, -0.02905191294848919, 0.04939625412225723, 0.07354278862476349, 0.04067093878984451, -0.013372430577874184, 0.008002068847417831, -0.10768777132034302, 0.047537051141262054, 0.015974529087543488, -0.008904634043574333, 0.023863598704338074, 0.021066976711153984, 0.050008613616228104, -0.029876243323087692, -0.002218670677393675, 0.07748720049858093, 0.023606780916452408, 0.04776189103722572, -0.043405383825302124, 0.018267665058374405, 0.007505507208406925, -0.0597020760178566, 0.025565888732671738, 0.06444871425628662, -0.04338037222623825, 0.05738864466547966, 0.037532441318035126, 0.02582477778196335, -0.0088923629373312, 0.051100559532642365], "cos_innovation_negg": [0.003765084780752659, 0.00043217861093580723, 0.0013481127098202705, 0.053083255887031555, 0.012091723270714283, 0.004633747041225433, 0.003966432996094227, -0.026057565584778786, -0.008876107633113861, 0.02871987782418728, 0.04269128292798996, 0.0014793600421398878, 0.025191795080900192, -0.0674225240945816, 0.015505467541515827, -0.06318938732147217, -0.031499456614255905, 0.07636164128780365, 0.023246653378009796, -0.0035817446187138557, 0.04152105748653412, -0.01521938107907772, -0.041440315544605255, 0.038943491876125336, 0.040427740663290024, 0.04755157232284546, 0.04602851718664169, -0.022689521312713623, -0.001713272649794817, -0.015236373990774155, -0.02905191294848919, 0.04939625412225723, 0.07354278862476349, 0.04067093878984451, -0.013372430577874184, 0.008002068847417831, -0.10768777132034302, 0.047537051141262054, 0.015974529087543488, -0.008904634043574333, 0.023863598704338074, 0.021066976711153984, 0.050008613616228104, -0.029876243323087692, -0.002218670677393675, 0.07748720049858093, 0.023606780916452408, 0.04776189103722572, -0.043405383825302124, 0.018267665058374405, 0.007505507208406925, -0.0597020760178566, 0.025565888732671738, 0.06444871425628662, -0.04338037222623825, 0.05738864466547966, 0.037532441318035126, 0.02582477778196335, -0.0088923629373312, 0.051100559532642365], "cos_apical_negg": [0.003765084780752659, 0.00043217861093580723, 0.0013481127098202705, 0.053083255887031555, 0.012091723270714283, 0.004633747041225433, 0.003966432996094227, -0.026057565584778786, -0.008876107633113861, 0.02871987782418728, 0.04269128292798996, 0.0014793600421398878, 0.025191795080900192, -0.0674225240945816, 0.015505467541515827, -0.06318938732147217, -0.031499456614255905, 0.07636164128780365, 0.023246653378009796, -0.0035817446187138557, 0.04152105748653412, -0.01521938107907772, -0.041440315544605255, 0.038943491876125336, 0.040427740663290024, 0.04755157232284546, 0.04602851718664169, -0.022689521312713623, -0.001713272649794817, -0.015236373990774155, -0.02905191294848919, 0.04939625412225723, 0.07354278862476349, 0.04067093878984451, -0.013372430577874184, 0.008002068847417831, -0.10768777132034302, 0.047537051141262054, 0.015974529087543488, -0.008904634043574333, 0.023863598704338074, 0.021066976711153984, 0.050008613616228104, -0.029876243323087692, -0.002218670677393675, 0.07748720049858093, 0.023606780916452408, 0.04776189103722572, -0.043405383825302124, 0.018267665058374405, 0.007505507208406925, -0.0597020760178566, 0.025565888732671738, 0.06444871425628662, -0.04338037222623825, 0.05738864466547966, 0.037532441318035126, 0.02582477778196335, -0.0088923629373312, 0.051100559532642365], "cos_Ac_negg": [0.003765084780752659, 0.00043217861093580723, 0.0013481127098202705, 0.053083255887031555, 0.012091723270714283, 0.004633747041225433, 0.003966432996094227, -0.026057565584778786, -0.008876107633113861, 0.02871987782418728, 0.04269128292798996, 0.0014793600421398878, 0.025191795080900192, -0.0674225240945816, 0.015505467541515827, -0.06318938732147217, -0.031499456614255905, 0.07636164128780365, 0.023246653378009796, -0.0035817446187138557, 0.04152105748653412, -0.01521938107907772, -0.041440315544605255, 0.038943491876125336, 0.040427740663290024, 0.04755157232284546, 0.04602851718664169, -0.022689521312713623, -0.001713272649794817, -0.015236373990774155, -0.02905191294848919, 0.04939625412225723, 0.07354278862476349, 0.04067093878984451, -0.013372430577874184, 0.008002068847417831, -0.10768777132034302, 0.047537051141262054, 0.015974529087543488, -0.008904634043574333, 0.023863598704338074, 0.021066976711153984, 0.050008613616228104, -0.029876243323087692, -0.002218670677393675, 0.07748720049858093, 0.023606780916452408, 0.04776189103722572, -0.043405383825302124, 0.018267665058374405, 0.007505507208406925, -0.0597020760178566, 0.025565888732671738, 0.06444871425628662, -0.04338037222623825, 0.05738864466547966, 0.037532441318035126, 0.02582477778196335, -0.0088923629373312, 0.051100559532642365], "r_norm": [2.522134304046631, 2.447347640991211, 2.350313186645508, 2.3809990882873535, 2.4430148601531982, 2.540632486343384, 2.3381285667419434, 2.532336711883545, 2.413891553878784, 2.429677724838257, 2.468790054321289, 2.2367753982543945, 2.3438096046447754, 2.5256733894348145, 2.439521551132202, 2.395617723464966, 2.51021671295166, 2.287209987640381, 2.439669609069824, 2.5274295806884766, 2.5747010707855225, 2.5228583812713623, 2.477553129196167, 2.573854446411133, 2.415081739425659, 2.410484552383423, 2.3728368282318115, 2.363971710205078, 2.4078867435455322, 2.5769596099853516, 2.518977642059326, 2.475651502609253, 2.4537532329559326, 2.4495491981506348, 2.451180934906006, 2.4466638565063477, 2.4685401916503906, 2.461303234100342, 2.47576904296875, 2.320603609085083, 2.4525327682495117, 2.4522624015808105, 2.426569700241089, 2.4643492698669434, 2.2672011852264404, 2.5280802249908447, 2.3629062175750732, 2.403710126876831, 2.5437235832214355, 2.517758846282959, 2.5039658546447754, 2.3951034545898438, 2.328484535217285, 2.387754440307617, 2.319270133972168, 2.3547277450561523, 2.4390511512756348, 2.393573760986328, 2.39512300491333, 2.481116771697998], "ldr": {"dL_sdil": 0.07580971717834473, "dL_bp": -0.3394639492034912, "ratio": -0.22332185009990765}}, {"epoch_end": 0, "step": 391, "test_acc": 0.365, "test_loss": 1.816493798828125}, {"step": 500, "epoch": 1, "train_loss": 1.6926206350326538, "cos_r_negg": [0.6016989946365356, 0.07451913505792618, 0.10846629738807678, 0.0791502445936203, 0.04639650881290436, 0.06966159492731094, 0.09633296728134155, 0.06501592695713043, -0.08884154260158539, 0.10408034920692444, 0.040592391043901443, 0.04684300720691681, 0.07506699860095978, 0.07191833853721619, 0.14936563372612, 0.05595159903168678, 0.0384545624256134, 0.08786694705486298, 0.027657005935907364, 0.10507886111736298, 0.08618521690368652, -0.01262473501265049, 0.07781089097261429, 0.01669483073055744, 0.0883541852235794, 0.06679975241422653, 0.08781743794679642, 0.03621674329042435, 0.11098434031009674, 0.04391704872250557, 0.05769789218902588, 0.08356957137584686, 0.04840085655450821, 0.08488386124372482, 0.08337865769863129, 0.05138924717903137, -0.006969223730266094, 0.08633480966091156, -0.05820207670331001, 0.1580529808998108, 0.047267574816942215, 0.08189140260219574, 0.048528313636779785, 0.010633558966219425, 0.06523486226797104, 0.0825146809220314, 0.05125541239976883, 0.08097868412733078, 0.05268680304288864, 0.01872938871383667, 0.06720461696386337, 0.04882297292351723, 0.05587758496403694, 0.07753224670886993, 0.02450636774301529, 0.015463694930076599, 0.09650469571352005, 0.03222125768661499, 0.021985560655593872, 0.1258968561887741], "cos_innovation_negg": [0.6016989946365356, 0.07451913505792618, 0.10846629738807678, 0.0791502445936203, 0.04639650881290436, 0.06966159492731094, 0.09633296728134155, 0.06501592695713043, -0.08884154260158539, 0.10408034920692444, 0.040592391043901443, 0.04684300720691681, 0.07506699860095978, 0.07191833853721619, 0.14936563372612, 0.05595159903168678, 0.0384545624256134, 0.08786694705486298, 0.027657005935907364, 0.10507886111736298, 0.08618521690368652, -0.01262473501265049, 0.07781089097261429, 0.01669483073055744, 0.0883541852235794, 0.06679975241422653, 0.08781743794679642, 0.03621674329042435, 0.11098434031009674, 0.04391704872250557, 0.05769789218902588, 0.08356957137584686, 0.04840085655450821, 0.08488386124372482, 0.08337865769863129, 0.05138924717903137, -0.006969223730266094, 0.08633480966091156, -0.05820207670331001, 0.1580529808998108, 0.047267574816942215, 0.08189140260219574, 0.048528313636779785, 0.010633558966219425, 0.06523486226797104, 0.0825146809220314, 0.05125541239976883, 0.08097868412733078, 0.05268680304288864, 0.01872938871383667, 0.06720461696386337, 0.04882297292351723, 0.05587758496403694, 0.07753224670886993, 0.02450636774301529, 0.015463694930076599, 0.09650469571352005, 0.03222125768661499, 0.021985560655593872, 0.1258968561887741], "cos_apical_negg": [0.6016989946365356, 0.07451913505792618, 0.10846629738807678, 0.0791502445936203, 0.04639650881290436, 0.06966159492731094, 0.09633296728134155, 0.06501592695713043, -0.08884154260158539, 0.10408034920692444, 0.040592391043901443, 0.04684300720691681, 0.07506699860095978, 0.07191833853721619, 0.14936563372612, 0.05595159903168678, 0.0384545624256134, 0.08786694705486298, 0.027657005935907364, 0.10507886111736298, 0.08618521690368652, -0.01262473501265049, 0.07781089097261429, 0.01669483073055744, 0.0883541852235794, 0.06679975241422653, 0.08781743794679642, 0.03621674329042435, 0.11098434031009674, 0.04391704872250557, 0.05769789218902588, 0.08356957137584686, 0.04840085655450821, 0.08488386124372482, 0.08337865769863129, 0.05138924717903137, -0.006969223730266094, 0.08633480966091156, -0.05820207670331001, 0.1580529808998108, 0.047267574816942215, 0.08189140260219574, 0.048528313636779785, 0.010633558966219425, 0.06523486226797104, 0.0825146809220314, 0.05125541239976883, 0.08097868412733078, 0.05268680304288864, 0.01872938871383667, 0.06720461696386337, 0.04882297292351723, 0.05587758496403694, 0.07753224670886993, 0.02450636774301529, 0.015463694930076599, 0.09650469571352005, 0.03222125768661499, 0.021985560655593872, 0.1258968561887741], "cos_Ac_negg": [0.6016989946365356, 0.07451913505792618, 0.10846629738807678, 0.0791502445936203, 0.04639650881290436, 0.06966159492731094, 0.09633296728134155, 0.06501592695713043, -0.08884154260158539, 0.10408034920692444, 0.040592391043901443, 0.04684300720691681, 0.07506699860095978, 0.07191833853721619, 0.14936563372612, 0.05595159903168678, 0.0384545624256134, 0.08786694705486298, 0.027657005935907364, 0.10507886111736298, 0.08618521690368652, -0.01262473501265049, 0.07781089097261429, 0.01669483073055744, 0.0883541852235794, 0.06679975241422653, 0.08781743794679642, 0.03621674329042435, 0.11098434031009674, 0.04391704872250557, 0.05769789218902588, 0.08356957137584686, 0.04840085655450821, 0.08488386124372482, 0.08337865769863129, 0.05138924717903137, -0.006969223730266094, 0.08633480966091156, -0.05820207670331001, 0.1580529808998108, 0.047267574816942215, 0.08189140260219574, 0.048528313636779785, 0.010633558966219425, 0.06523486226797104, 0.0825146809220314, 0.05125541239976883, 0.08097868412733078, 0.05268680304288864, 0.01872938871383667, 0.06720461696386337, 0.04882297292351723, 0.05587758496403694, 0.07753224670886993, 0.02450636774301529, 0.015463694930076599, 0.09650469571352005, 0.03222125768661499, 0.021985560655593872, 0.1258968561887741], "r_norm": [1.8769549131393433, 1.7614719867706299, 1.6675233840942383, 1.7233858108520508, 1.720156192779541, 1.8139724731445312, 1.6855320930480957, 1.8172810077667236, 1.7339825630187988, 1.7434077262878418, 1.7531135082244873, 1.6072609424591064, 1.6773501634597778, 1.792986273765564, 1.735316276550293, 1.6980236768722534, 1.781245231628418, 1.6591107845306396, 1.74596107006073, 1.7987704277038574, 1.8486900329589844, 1.798914909362793, 1.755151629447937, 1.8445427417755127, 1.7317187786102295, 1.7446768283843994, 1.700711727142334, 1.6781278848648071, 1.7235705852508545, 1.8291743993759155, 1.8114674091339111, 1.7743232250213623, 1.7589789628982544, 1.7667059898376465, 1.7345802783966064, 1.7413233518600464, 1.76577627658844, 1.7622110843658447, 1.764195203781128, 1.6838276386260986, 1.7508625984191895, 1.7678618431091309, 1.7153913974761963, 1.753913164138794, 1.603890299797058, 1.8053665161132812, 1.6955610513687134, 1.7146875858306885, 1.7984004020690918, 1.7925623655319214, 1.8009240627288818, 1.7180745601654053, 1.6764795780181885, 1.7158640623092651, 1.6592435836791992, 1.688051462173462, 1.7396972179412842, 1.699596881866455, 1.7132997512817383, 1.7743945121765137]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3881, "test_loss": 1.76047431640625}, {"step": 1000, "epoch": 2, "train_loss": 1.7120805978775024, "cos_r_negg": [0.7038990259170532, 0.1557237058877945, 0.18351954221725464, 0.08345353603363037, 0.10455122590065002, 0.0859462097287178, 0.12286284565925598, 0.10458654165267944, 0.03384546563029289, 0.16147872805595398, 0.11091583222150803, 0.10411569476127625, 0.1296410858631134, 0.12392119318246841, 0.1895999163389206, 0.1119234561920166, 0.06548453867435455, 0.1396988034248352, 0.13002893328666687, 0.12539537250995636, 0.08902395516633987, 0.04449225589632988, 0.12689992785453796, 0.07941284775733948, 0.14808599650859833, 0.09376388788223267, 0.1616687774658203, 0.10233548283576965, 0.167471244931221, 0.12404613196849823, 0.09661027789115906, 0.13481149077415466, 0.11366836726665497, 0.15426142513751984, 0.128334641456604, 0.12535148859024048, 0.025407738983631134, 0.14866678416728973, 0.048488713800907135, 0.17955011129379272, 0.08145958185195923, 0.13605688512325287, 0.11944523453712463, 0.09732738137245178, 0.15545973181724548, 0.12466219067573547, 0.08548195660114288, 0.16475391387939453, 0.10165302455425262, 0.08624003082513809, 0.12783384323120117, 0.13885897397994995, 0.11248652637004852, 0.1251433938741684, 0.12427424639463425, 0.07538698613643646, 0.16443407535552979, 0.10328369587659836, 0.13630908727645874, 0.1460983157157898], "cos_innovation_negg": [0.7038990259170532, 0.1557237058877945, 0.18351954221725464, 0.08345353603363037, 0.10455122590065002, 0.0859462097287178, 0.12286284565925598, 0.10458654165267944, 0.03384546563029289, 0.16147872805595398, 0.11091583222150803, 0.10411569476127625, 0.1296410858631134, 0.12392119318246841, 0.1895999163389206, 0.1119234561920166, 0.06548453867435455, 0.1396988034248352, 0.13002893328666687, 0.12539537250995636, 0.08902395516633987, 0.04449225589632988, 0.12689992785453796, 0.07941284775733948, 0.14808599650859833, 0.09376388788223267, 0.1616687774658203, 0.10233548283576965, 0.167471244931221, 0.12404613196849823, 0.09661027789115906, 0.13481149077415466, 0.11366836726665497, 0.15426142513751984, 0.128334641456604, 0.12535148859024048, 0.025407738983631134, 0.14866678416728973, 0.048488713800907135, 0.17955011129379272, 0.08145958185195923, 0.13605688512325287, 0.11944523453712463, 0.09732738137245178, 0.15545973181724548, 0.12466219067573547, 0.08548195660114288, 0.16475391387939453, 0.10165302455425262, 0.08624003082513809, 0.12783384323120117, 0.13885897397994995, 0.11248652637004852, 0.1251433938741684, 0.12427424639463425, 0.07538698613643646, 0.16443407535552979, 0.10328369587659836, 0.13630908727645874, 0.1460983157157898], "cos_apical_negg": [0.7038990259170532, 0.1557237058877945, 0.18351954221725464, 0.08345353603363037, 0.10455122590065002, 0.0859462097287178, 0.12286284565925598, 0.10458654165267944, 0.03384546563029289, 0.16147872805595398, 0.11091583222150803, 0.10411569476127625, 0.1296410858631134, 0.12392119318246841, 0.1895999163389206, 0.1119234561920166, 0.06548453867435455, 0.1396988034248352, 0.13002893328666687, 0.12539537250995636, 0.08902395516633987, 0.04449225589632988, 0.12689992785453796, 0.07941284775733948, 0.14808599650859833, 0.09376388788223267, 0.1616687774658203, 0.10233548283576965, 0.167471244931221, 0.12404613196849823, 0.09661027789115906, 0.13481149077415466, 0.11366836726665497, 0.15426142513751984, 0.128334641456604, 0.12535148859024048, 0.025407738983631134, 0.14866678416728973, 0.048488713800907135, 0.17955011129379272, 0.08145958185195923, 0.13605688512325287, 0.11944523453712463, 0.09732738137245178, 0.15545973181724548, 0.12466219067573547, 0.08548195660114288, 0.16475391387939453, 0.10165302455425262, 0.08624003082513809, 0.12783384323120117, 0.13885897397994995, 0.11248652637004852, 0.1251433938741684, 0.12427424639463425, 0.07538698613643646, 0.16443407535552979, 0.10328369587659836, 0.13630908727645874, 0.1460983157157898], "cos_Ac_negg": [0.7038990259170532, 0.1557237058877945, 0.18351954221725464, 0.08345353603363037, 0.10455122590065002, 0.0859462097287178, 0.12286284565925598, 0.10458654165267944, 0.03384546563029289, 0.16147872805595398, 0.11091583222150803, 0.10411569476127625, 0.1296410858631134, 0.12392119318246841, 0.1895999163389206, 0.1119234561920166, 0.06548453867435455, 0.1396988034248352, 0.13002893328666687, 0.12539537250995636, 0.08902395516633987, 0.04449225589632988, 0.12689992785453796, 0.07941284775733948, 0.14808599650859833, 0.09376388788223267, 0.1616687774658203, 0.10233548283576965, 0.167471244931221, 0.12404613196849823, 0.09661027789115906, 0.13481149077415466, 0.11366836726665497, 0.15426142513751984, 0.128334641456604, 0.12535148859024048, 0.025407738983631134, 0.14866678416728973, 0.048488713800907135, 0.17955011129379272, 0.08145958185195923, 0.13605688512325287, 0.11944523453712463, 0.09732738137245178, 0.15545973181724548, 0.12466219067573547, 0.08548195660114288, 0.16475391387939453, 0.10165302455425262, 0.08624003082513809, 0.12783384323120117, 0.13885897397994995, 0.11248652637004852, 0.1251433938741684, 0.12427424639463425, 0.07538698613643646, 0.16443407535552979, 0.10328369587659836, 0.13630908727645874, 0.1460983157157898], "r_norm": [1.5961697101593018, 1.4193177223205566, 1.3458794355392456, 1.38655424118042, 1.3861087560653687, 1.4494889974594116, 1.3623077869415283, 1.4641149044036865, 1.3727421760559082, 1.4154624938964844, 1.4118911027908325, 1.2923901081085205, 1.3582203388214111, 1.4472887516021729, 1.4153621196746826, 1.3667113780975342, 1.4248664379119873, 1.3375461101531982, 1.4036669731140137, 1.455678939819336, 1.4887803792953491, 1.4424433708190918, 1.4117376804351807, 1.470617651939392, 1.392202615737915, 1.4003409147262573, 1.3679554462432861, 1.3497241735458374, 1.3935017585754395, 1.4726247787475586, 1.4523433446884155, 1.4310390949249268, 1.4053843021392822, 1.4238368272781372, 1.3954362869262695, 1.3957723379135132, 1.4046988487243652, 1.4162585735321045, 1.4158843755722046, 1.355381965637207, 1.40324068069458, 1.4230268001556396, 1.3828023672103882, 1.408191442489624, 1.2993932962417603, 1.455148696899414, 1.3608759641647339, 1.3821570873260498, 1.4452203512191772, 1.4282360076904297, 1.4479291439056396, 1.3764281272888184, 1.341192603111267, 1.3858776092529297, 1.3332300186157227, 1.3532711267471313, 1.4091730117797852, 1.3636982440948486, 1.3788691759109497, 1.4366693496704102]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3943, "test_loss": 1.739713623046875}, {"step": 1500, "epoch": 3, "train_loss": 1.6653847694396973, "cos_r_negg": [0.7505210638046265, 0.26627445220947266, 0.252509206533432, 0.20116432011127472, 0.1957859992980957, 0.1827055811882019, 0.23622117936611176, 0.23171845078468323, 0.09095349162817001, 0.22141286730766296, 0.18194834887981415, 0.19965602457523346, 0.22851908206939697, 0.22650329768657684, 0.2868276536464691, 0.2188744843006134, 0.1846741884946823, 0.20206020772457123, 0.22034671902656555, 0.21747058629989624, 0.1772858053445816, 0.12357445061206818, 0.21155306696891785, 0.13954533636569977, 0.22062964737415314, 0.17326697707176208, 0.24927426874637604, 0.20004473626613617, 0.2102276235818863, 0.21542498469352722, 0.17997604608535767, 0.21378371119499207, 0.16171492636203766, 0.2649158239364624, 0.2263120710849762, 0.18616516888141632, 0.11820511519908905, 0.25375938415527344, 0.13931803405284882, 0.25458893179893494, 0.12120676040649414, 0.2033802568912506, 0.23027682304382324, 0.18797756731510162, 0.2137695848941803, 0.19403722882270813, 0.18967431783676147, 0.2632417678833008, 0.16509349644184113, 0.18088336288928986, 0.21580520272254944, 0.2504417300224304, 0.22007392346858978, 0.2108559012413025, 0.24859291315078735, 0.19644981622695923, 0.2699081301689148, 0.19991938769817352, 0.240191251039505, 0.2573099136352539], "cos_innovation_negg": [0.7505210638046265, 0.26627445220947266, 0.252509206533432, 0.20116432011127472, 0.1957859992980957, 0.1827055811882019, 0.23622117936611176, 0.23171845078468323, 0.09095349162817001, 0.22141286730766296, 0.18194834887981415, 0.19965602457523346, 0.22851908206939697, 0.22650329768657684, 0.2868276536464691, 0.2188744843006134, 0.1846741884946823, 0.20206020772457123, 0.22034671902656555, 0.21747058629989624, 0.1772858053445816, 0.12357445061206818, 0.21155306696891785, 0.13954533636569977, 0.22062964737415314, 0.17326697707176208, 0.24927426874637604, 0.20004473626613617, 0.2102276235818863, 0.21542498469352722, 0.17997604608535767, 0.21378371119499207, 0.16171492636203766, 0.2649158239364624, 0.2263120710849762, 0.18616516888141632, 0.11820511519908905, 0.25375938415527344, 0.13931803405284882, 0.25458893179893494, 0.12120676040649414, 0.2033802568912506, 0.23027682304382324, 0.18797756731510162, 0.2137695848941803, 0.19403722882270813, 0.18967431783676147, 0.2632417678833008, 0.16509349644184113, 0.18088336288928986, 0.21580520272254944, 0.2504417300224304, 0.22007392346858978, 0.2108559012413025, 0.24859291315078735, 0.19644981622695923, 0.2699081301689148, 0.19991938769817352, 0.240191251039505, 0.2573099136352539], "cos_apical_negg": [0.7505210638046265, 0.26627445220947266, 0.252509206533432, 0.20116432011127472, 0.1957859992980957, 0.1827055811882019, 0.23622117936611176, 0.23171845078468323, 0.09095349162817001, 0.22141286730766296, 0.18194834887981415, 0.19965602457523346, 0.22851908206939697, 0.22650329768657684, 0.2868276536464691, 0.2188744843006134, 0.1846741884946823, 0.20206020772457123, 0.22034671902656555, 0.21747058629989624, 0.1772858053445816, 0.12357445061206818, 0.21155306696891785, 0.13954533636569977, 0.22062964737415314, 0.17326697707176208, 0.24927426874637604, 0.20004473626613617, 0.2102276235818863, 0.21542498469352722, 0.17997604608535767, 0.21378371119499207, 0.16171492636203766, 0.2649158239364624, 0.2263120710849762, 0.18616516888141632, 0.11820511519908905, 0.25375938415527344, 0.13931803405284882, 0.25458893179893494, 0.12120676040649414, 0.2033802568912506, 0.23027682304382324, 0.18797756731510162, 0.2137695848941803, 0.19403722882270813, 0.18967431783676147, 0.2632417678833008, 0.16509349644184113, 0.18088336288928986, 0.21580520272254944, 0.2504417300224304, 0.22007392346858978, 0.2108559012413025, 0.24859291315078735, 0.19644981622695923, 0.2699081301689148, 0.19991938769817352, 0.240191251039505, 0.2573099136352539], "cos_Ac_negg": [0.7505210638046265, 0.26627445220947266, 0.252509206533432, 0.20116432011127472, 0.1957859992980957, 0.1827055811882019, 0.23622117936611176, 0.23171845078468323, 0.09095349162817001, 0.22141286730766296, 0.18194834887981415, 0.19965602457523346, 0.22851908206939697, 0.22650329768657684, 0.2868276536464691, 0.2188744843006134, 0.1846741884946823, 0.20206020772457123, 0.22034671902656555, 0.21747058629989624, 0.1772858053445816, 0.12357445061206818, 0.21155306696891785, 0.13954533636569977, 0.22062964737415314, 0.17326697707176208, 0.24927426874637604, 0.20004473626613617, 0.2102276235818863, 0.21542498469352722, 0.17997604608535767, 0.21378371119499207, 0.16171492636203766, 0.2649158239364624, 0.2263120710849762, 0.18616516888141632, 0.11820511519908905, 0.25375938415527344, 0.13931803405284882, 0.25458893179893494, 0.12120676040649414, 0.2033802568912506, 0.23027682304382324, 0.18797756731510162, 0.2137695848941803, 0.19403722882270813, 0.18967431783676147, 0.2632417678833008, 0.16509349644184113, 0.18088336288928986, 0.21580520272254944, 0.2504417300224304, 0.22007392346858978, 0.2108559012413025, 0.24859291315078735, 0.19644981622695923, 0.2699081301689148, 0.19991938769817352, 0.240191251039505, 0.2573099136352539], "r_norm": [1.3829537630081177, 1.1714832782745361, 1.1202938556671143, 1.1273932456970215, 1.1282145977020264, 1.1836298704147339, 1.1248453855514526, 1.193600058555603, 1.1210159063339233, 1.156036138534546, 1.1509356498718262, 1.0533576011657715, 1.110054850578308, 1.171433925628662, 1.1631346940994263, 1.1232197284698486, 1.1587131023406982, 1.094006061553955, 1.148818016052246, 1.1874732971191406, 1.2104703187942505, 1.1739047765731812, 1.1587660312652588, 1.197554588317871, 1.1408212184906006, 1.1399563550949097, 1.1281828880310059, 1.1054835319519043, 1.1471459865570068, 1.2002085447311401, 1.183532476425171, 1.1781301498413086, 1.149897813796997, 1.1677930355072021, 1.1475963592529297, 1.1412761211395264, 1.1385844945907593, 1.1653268337249756, 1.1500465869903564, 1.1192861795425415, 1.1380295753479004, 1.1681016683578491, 1.1400260925292969, 1.1458760499954224, 1.0682353973388672, 1.1891491413116455, 1.1102821826934814, 1.1420952081680298, 1.1780734062194824, 1.1705150604248047, 1.176927089691162, 1.1324958801269531, 1.095231294631958, 1.1394319534301758, 1.0962517261505127, 1.1041277647018433, 1.153808355331421, 1.1224653720855713, 1.1333494186401367, 1.1817761659622192]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.4038, "test_loss": 1.7170080444335938}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4155, "test_loss": 1.704402490234375}], "final": {"test_acc": 0.4155, "test_loss": 1.704402490234375, "wall_s": 87.50506520271301, "cos_r_negg": [0.8006995916366577, 0.3202219307422638, 0.33933261036872864, 0.2935324013233185, 0.3127063810825348, 0.28089505434036255, 0.31517964601516724, 0.3234487771987915, 0.18885204195976257, 0.2889353334903717, 0.2774583399295807, 0.3245980143547058, 0.29574117064476013, 0.31794172525405884, 0.3526517152786255, 0.3134472966194153, 0.27730828523635864, 0.33552196621894836, 0.33940398693084717, 0.31116822361946106, 0.26693785190582275, 0.20562313497066498, 0.3235408663749695, 0.22658710181713104, 0.3064729571342468, 0.2906871438026428, 0.35716554522514343, 0.3015293478965759, 0.34330761432647705, 0.2823939919471741, 0.27086105942726135, 0.3187156319618225, 0.28391847014427185, 0.3373302221298218, 0.3383590281009674, 0.28575220704078674, 0.24246099591255188, 0.3272182047367096, 0.22819814085960388, 0.38842064142227173, 0.2526533603668213, 0.299740731716156, 0.3685492277145386, 0.2869250774383545, 0.34325069189071655, 0.3374507427215576, 0.28348681330680847, 0.3438063859939575, 0.28782862424850464, 0.2886790633201599, 0.3348701000213623, 0.35544541478157043, 0.32329580187797546, 0.30473971366882324, 0.3617377281188965, 0.31999343633651733, 0.3497810959815979, 0.3082476854324341, 0.3598722815513611, 0.33828940987586975], "cos_innovation_negg": [0.8006995916366577, 0.3202219307422638, 0.33933261036872864, 0.2935324013233185, 0.3127063810825348, 0.28089505434036255, 0.31517964601516724, 0.3234487771987915, 0.18885204195976257, 0.2889353334903717, 0.2774583399295807, 0.3245980143547058, 0.29574117064476013, 0.31794172525405884, 0.3526517152786255, 0.3134472966194153, 0.27730828523635864, 0.33552196621894836, 0.33940398693084717, 0.31116822361946106, 0.26693785190582275, 0.20562313497066498, 0.3235408663749695, 0.22658710181713104, 0.3064729571342468, 0.2906871438026428, 0.35716554522514343, 0.3015293478965759, 0.34330761432647705, 0.2823939919471741, 0.27086105942726135, 0.3187156319618225, 0.28391847014427185, 0.3373302221298218, 0.3383590281009674, 0.28575220704078674, 0.24246099591255188, 0.3272182047367096, 0.22819814085960388, 0.38842064142227173, 0.2526533603668213, 0.299740731716156, 0.3685492277145386, 0.2869250774383545, 0.34325069189071655, 0.3374507427215576, 0.28348681330680847, 0.3438063859939575, 0.28782862424850464, 0.2886790633201599, 0.3348701000213623, 0.35544541478157043, 0.32329580187797546, 0.30473971366882324, 0.3617377281188965, 0.31999343633651733, 0.3497810959815979, 0.3082476854324341, 0.3598722815513611, 0.33828940987586975], "cos_apical_negg": [0.8006995916366577, 0.3202219307422638, 0.33933261036872864, 0.2935324013233185, 0.3127063810825348, 0.28089505434036255, 0.31517964601516724, 0.3234487771987915, 0.18885204195976257, 0.2889353334903717, 0.2774583399295807, 0.3245980143547058, 0.29574117064476013, 0.31794172525405884, 0.3526517152786255, 0.3134472966194153, 0.27730828523635864, 0.33552196621894836, 0.33940398693084717, 0.31116822361946106, 0.26693785190582275, 0.20562313497066498, 0.3235408663749695, 0.22658710181713104, 0.3064729571342468, 0.2906871438026428, 0.35716554522514343, 0.3015293478965759, 0.34330761432647705, 0.2823939919471741, 0.27086105942726135, 0.3187156319618225, 0.28391847014427185, 0.3373302221298218, 0.3383590281009674, 0.28575220704078674, 0.24246099591255188, 0.3272182047367096, 0.22819814085960388, 0.38842064142227173, 0.2526533603668213, 0.299740731716156, 0.3685492277145386, 0.2869250774383545, 0.34325069189071655, 0.3374507427215576, 0.28348681330680847, 0.3438063859939575, 0.28782862424850464, 0.2886790633201599, 0.3348701000213623, 0.35544541478157043, 0.32329580187797546, 0.30473971366882324, 0.3617377281188965, 0.31999343633651733, 0.3497810959815979, 0.3082476854324341, 0.3598722815513611, 0.33828940987586975], "cos_Ac_negg": [0.8006995916366577, 0.3202219307422638, 0.33933261036872864, 0.2935324013233185, 0.3127063810825348, 0.28089505434036255, 0.31517964601516724, 0.3234487771987915, 0.18885204195976257, 0.2889353334903717, 0.2774583399295807, 0.3245980143547058, 0.29574117064476013, 0.31794172525405884, 0.3526517152786255, 0.3134472966194153, 0.27730828523635864, 0.33552196621894836, 0.33940398693084717, 0.31116822361946106, 0.26693785190582275, 0.20562313497066498, 0.3235408663749695, 0.22658710181713104, 0.3064729571342468, 0.2906871438026428, 0.35716554522514343, 0.3015293478965759, 0.34330761432647705, 0.2823939919471741, 0.27086105942726135, 0.3187156319618225, 0.28391847014427185, 0.3373302221298218, 0.3383590281009674, 0.28575220704078674, 0.24246099591255188, 0.3272182047367096, 0.22819814085960388, 0.38842064142227173, 0.2526533603668213, 0.299740731716156, 0.3685492277145386, 0.2869250774383545, 0.34325069189071655, 0.3374507427215576, 0.28348681330680847, 0.3438063859939575, 0.28782862424850464, 0.2886790633201599, 0.3348701000213623, 0.35544541478157043, 0.32329580187797546, 0.30473971366882324, 0.3617377281188965, 0.31999343633651733, 0.3497810959815979, 0.3082476854324341, 0.3598722815513611, 0.33828940987586975]}} \ No newline at end of file diff --git a/results/scale_v3_cifar10_sdil_w64_d60_s4.json b/results/scale_v3_cifar10_sdil_w64_d60_s4.json new file mode 100644 index 0000000..2790c81 --- /dev/null +++ b/results/scale_v3_cifar10_sdil_w64_d60_s4.json @@ -0,0 +1 @@ +{"args": {"mode": "sdil", "dataset": "cifar10", "depth": 60, "width": 64, "act": "tanh", "residual": 1, "epochs": 5, "batch_size": 128, "eta": 0.02, "eta_A": 0.02, "eta_P": 0.002, "momentum": 0.9, "w_scale": 1.0, "a_scale": 1.0, "pert_sigma": 0.01, "pert_every": 4, "pert_ndirs": 16, "pert_mode": "simultaneous", "use_residual": 1, "raw_scale_control": "none", "learn_A": 1, "learn_P": 1, "p_neutral": 1, "p_warmup_steps": 200, "p_warmup_eta": 0.05, "nuis_rho": 0.0, "predictor_mode": "diagonal", "normalize_delta": 0, "settle_steps": 0, "kappa": 0.0, "feedback": "error", "seed": 4, "device": "cuda", "log_every": 500, "max_steps": 0, "probe_bs": 256, "outdir": "results", "tag": "scale_v3_cifar10_sdil_w64_d60_s4", "n_in": 3072}, "provenance": {"git_commit": "c8a372cae2612839bdbdafcf9c114f52cb4a6de3", "git_dirty": false}, "steps": [{"step": 0, "epoch": 0, "train_loss": 2.6918692588806152, "cos_r_negg": [-0.04490283504128456, -0.010329673998057842, -0.06371335685253143, -0.019623106345534325, -0.037642188370227814, 0.055330365896224976, 0.0018151490949094296, -0.10804343223571777, 0.04364018142223358, 0.046805839985609055, 0.02182871662080288, -0.06083410978317261, -0.009318496100604534, 0.015976140275597572, -0.0063055045902729034, 0.005943970754742622, 0.0022731178905814886, -0.024580752477049828, 0.02548070065677166, 0.050730880349874496, -0.035578057169914246, 0.01839134283363819, -0.0648098811507225, 0.061392150819301605, -0.019604893401265144, -0.06565262377262115, -0.03610725328326225, -0.005412286147475243, -0.06614495813846588, 0.04097066819667816, 0.0037323334254324436, 0.03230953589081764, 0.07421408593654633, -0.026431303471326828, -0.018891436979174614, 0.0072016529738903046, 0.052864428609609604, -0.03291391581296921, 0.003911101724952459, -0.013009095564484596, -0.046503759920597076, 0.017538229003548622, -0.01203260850161314, 0.0535823255777359, -0.026532433927059174, -0.04576018080115318, 0.0026301974430680275, 0.02484653890132904, -0.04644635319709778, 0.003194622229784727, 0.014532333239912987, 0.025225674733519554, -0.024414192885160446, 0.015294492244720459, 0.10216866433620453, 0.030490782111883163, 0.09271576255559921, -0.015625599771738052, -0.11982617527246475, -0.013249961659312248], "cos_innovation_negg": [-0.04490283504128456, -0.010329673998057842, -0.06371335685253143, -0.019623106345534325, -0.037642188370227814, 0.055330365896224976, 0.0018151490949094296, -0.10804343223571777, 0.04364018142223358, 0.046805839985609055, 0.02182871662080288, -0.06083410978317261, -0.009318496100604534, 0.015976140275597572, -0.0063055045902729034, 0.005943970754742622, 0.0022731178905814886, -0.024580752477049828, 0.02548070065677166, 0.050730880349874496, -0.035578057169914246, 0.01839134283363819, -0.0648098811507225, 0.061392150819301605, -0.019604893401265144, -0.06565262377262115, -0.03610725328326225, -0.005412286147475243, -0.06614495813846588, 0.04097066819667816, 0.0037323334254324436, 0.03230953589081764, 0.07421408593654633, -0.026431303471326828, -0.018891436979174614, 0.0072016529738903046, 0.052864428609609604, -0.03291391581296921, 0.003911101724952459, -0.013009095564484596, -0.046503759920597076, 0.017538229003548622, -0.01203260850161314, 0.0535823255777359, -0.026532433927059174, -0.04576018080115318, 0.0026301974430680275, 0.02484653890132904, -0.04644635319709778, 0.003194622229784727, 0.014532333239912987, 0.025225674733519554, -0.024414192885160446, 0.015294492244720459, 0.10216866433620453, 0.030490782111883163, 0.09271576255559921, -0.015625599771738052, -0.11982617527246475, -0.013249961659312248], "cos_apical_negg": [-0.04490283504128456, -0.010329673998057842, -0.06371335685253143, -0.019623106345534325, -0.037642188370227814, 0.055330365896224976, 0.0018151490949094296, -0.10804343223571777, 0.04364018142223358, 0.046805839985609055, 0.02182871662080288, -0.06083410978317261, -0.009318496100604534, 0.015976140275597572, -0.0063055045902729034, 0.005943970754742622, 0.0022731178905814886, -0.024580752477049828, 0.02548070065677166, 0.050730880349874496, -0.035578057169914246, 0.01839134283363819, -0.0648098811507225, 0.061392150819301605, -0.019604893401265144, -0.06565262377262115, -0.03610725328326225, -0.005412286147475243, -0.06614495813846588, 0.04097066819667816, 0.0037323334254324436, 0.03230953589081764, 0.07421408593654633, -0.026431303471326828, -0.018891436979174614, 0.0072016529738903046, 0.052864428609609604, -0.03291391581296921, 0.003911101724952459, -0.013009095564484596, -0.046503759920597076, 0.017538229003548622, -0.01203260850161314, 0.0535823255777359, -0.026532433927059174, -0.04576018080115318, 0.0026301974430680275, 0.02484653890132904, -0.04644635319709778, 0.003194622229784727, 0.014532333239912987, 0.025225674733519554, -0.024414192885160446, 0.015294492244720459, 0.10216866433620453, 0.030490782111883163, 0.09271576255559921, -0.015625599771738052, -0.11982617527246475, -0.013249961659312248], "cos_Ac_negg": [-0.04490283504128456, -0.010329673998057842, -0.06371335685253143, -0.019623106345534325, -0.037642188370227814, 0.055330365896224976, 0.0018151490949094296, -0.10804343223571777, 0.04364018142223358, 0.046805839985609055, 0.02182871662080288, -0.06083410978317261, -0.009318496100604534, 0.015976140275597572, -0.0063055045902729034, 0.005943970754742622, 0.0022731178905814886, -0.024580752477049828, 0.02548070065677166, 0.050730880349874496, -0.035578057169914246, 0.01839134283363819, -0.0648098811507225, 0.061392150819301605, -0.019604893401265144, -0.06565262377262115, -0.03610725328326225, -0.005412286147475243, -0.06614495813846588, 0.04097066819667816, 0.0037323334254324436, 0.03230953589081764, 0.07421408593654633, -0.026431303471326828, -0.018891436979174614, 0.0072016529738903046, 0.052864428609609604, -0.03291391581296921, 0.003911101724952459, -0.013009095564484596, -0.046503759920597076, 0.017538229003548622, -0.01203260850161314, 0.0535823255777359, -0.026532433927059174, -0.04576018080115318, 0.0026301974430680275, 0.02484653890132904, -0.04644635319709778, 0.003194622229784727, 0.014532333239912987, 0.025225674733519554, -0.024414192885160446, 0.015294492244720459, 0.10216866433620453, 0.030490782111883163, 0.09271576255559921, -0.015625599771738052, -0.11982617527246475, -0.013249961659312248], "r_norm": [2.566884994506836, 2.519351005554199, 2.569491147994995, 2.415595769882202, 2.4288196563720703, 2.4834794998168945, 2.437126874923706, 2.4399900436401367, 2.4605329036712646, 2.4892168045043945, 2.4857840538024902, 2.464425802230835, 2.459109306335449, 2.3497555255889893, 2.404336929321289, 2.510669231414795, 2.5845937728881836, 2.439289093017578, 2.468700408935547, 2.5426125526428223, 2.522217273712158, 2.6081080436706543, 2.399277448654175, 2.57711124420166, 2.3723621368408203, 2.4417130947113037, 2.3620476722717285, 2.4193851947784424, 2.54929780960083, 2.5581307411193848, 2.359800338745117, 2.403446674346924, 2.476604461669922, 2.6039376258850098, 2.3283023834228516, 2.458747625350952, 2.6523797512054443, 2.4621574878692627, 2.557375907897949, 2.4677295684814453, 2.47267484664917, 2.4618096351623535, 2.5432662963867188, 2.3896634578704834, 2.4328277111053467, 2.4771268367767334, 2.500183582305908, 2.457491874694824, 2.360447406768799, 2.4968724250793457, 2.325652599334717, 2.4409263134002686, 2.4057869911193848, 2.3778879642486572, 2.375293254852295, 2.507054328918457, 2.533440351486206, 2.464116096496582, 2.526402473449707, 2.3758351802825928], "ldr": {"dL_sdil": -0.06161832809448242, "dL_bp": -0.4646170139312744, "ratio": 0.13262176426366712}}, {"epoch_end": 0, "step": 391, "test_acc": 0.3675, "test_loss": 1.80766171875}, {"step": 500, "epoch": 1, "train_loss": 1.6820752620697021, "cos_r_negg": [0.6272779107093811, 0.09554404020309448, 0.08841381222009659, 0.03544314578175545, 0.058252446353435516, 0.09822554141283035, 0.04386086389422417, 0.051703598350286484, 0.0914786159992218, 0.10188747197389603, 0.03786192834377289, 0.03893250972032547, 0.041348159313201904, 0.10928703099489212, 0.004410627298057079, 0.12595109641551971, 0.04410814866423607, 0.08702585101127625, 0.02068858966231346, 1.6868114471435547e-05, 0.03216884657740593, 0.02092152275145054, 0.029736410826444626, -0.01660814695060253, 0.10467110574245453, 0.07446694374084473, 0.03742540627717972, -0.04454159736633301, 0.0001198919489979744, -0.010586773976683617, 0.06511332094669342, 0.07579436898231506, 0.07500766962766647, 0.004093728959560394, 0.08108247816562653, 0.08558335900306702, 0.08037269115447998, 0.084686279296875, 0.06961989402770996, 0.10539840161800385, 0.030324822291731834, 0.08596324920654297, 0.13491132855415344, 0.1088460311293602, 0.09309998899698257, 0.035371311008930206, 0.07957285642623901, -0.0028438279405236244, -0.04044720530509949, 0.05673590302467346, 0.0667867511510849, 0.06647487729787827, 0.09840860962867737, 0.01284649409353733, 0.0485856756567955, 0.1468493640422821, 0.13607463240623474, 0.0837700366973877, 0.006812230218201876, 0.02709188312292099], "cos_innovation_negg": [0.6272779107093811, 0.09554404020309448, 0.08841381222009659, 0.03544314578175545, 0.058252446353435516, 0.09822554141283035, 0.04386086389422417, 0.051703598350286484, 0.0914786159992218, 0.10188747197389603, 0.03786192834377289, 0.03893250972032547, 0.041348159313201904, 0.10928703099489212, 0.004410627298057079, 0.12595109641551971, 0.04410814866423607, 0.08702585101127625, 0.02068858966231346, 1.6868114471435547e-05, 0.03216884657740593, 0.02092152275145054, 0.029736410826444626, -0.01660814695060253, 0.10467110574245453, 0.07446694374084473, 0.03742540627717972, -0.04454159736633301, 0.0001198919489979744, -0.010586773976683617, 0.06511332094669342, 0.07579436898231506, 0.07500766962766647, 0.004093728959560394, 0.08108247816562653, 0.08558335900306702, 0.08037269115447998, 0.084686279296875, 0.06961989402770996, 0.10539840161800385, 0.030324822291731834, 0.08596324920654297, 0.13491132855415344, 0.1088460311293602, 0.09309998899698257, 0.035371311008930206, 0.07957285642623901, -0.0028438279405236244, -0.04044720530509949, 0.05673590302467346, 0.0667867511510849, 0.06647487729787827, 0.09840860962867737, 0.01284649409353733, 0.0485856756567955, 0.1468493640422821, 0.13607463240623474, 0.0837700366973877, 0.006812230218201876, 0.02709188312292099], "cos_apical_negg": [0.6272779107093811, 0.09554404020309448, 0.08841381222009659, 0.03544314578175545, 0.058252446353435516, 0.09822554141283035, 0.04386086389422417, 0.051703598350286484, 0.0914786159992218, 0.10188747197389603, 0.03786192834377289, 0.03893250972032547, 0.041348159313201904, 0.10928703099489212, 0.004410627298057079, 0.12595109641551971, 0.04410814866423607, 0.08702585101127625, 0.02068858966231346, 1.6868114471435547e-05, 0.03216884657740593, 0.02092152275145054, 0.029736410826444626, -0.01660814695060253, 0.10467110574245453, 0.07446694374084473, 0.03742540627717972, -0.04454159736633301, 0.0001198919489979744, -0.010586773976683617, 0.06511332094669342, 0.07579436898231506, 0.07500766962766647, 0.004093728959560394, 0.08108247816562653, 0.08558335900306702, 0.08037269115447998, 0.084686279296875, 0.06961989402770996, 0.10539840161800385, 0.030324822291731834, 0.08596324920654297, 0.13491132855415344, 0.1088460311293602, 0.09309998899698257, 0.035371311008930206, 0.07957285642623901, -0.0028438279405236244, -0.04044720530509949, 0.05673590302467346, 0.0667867511510849, 0.06647487729787827, 0.09840860962867737, 0.01284649409353733, 0.0485856756567955, 0.1468493640422821, 0.13607463240623474, 0.0837700366973877, 0.006812230218201876, 0.02709188312292099], "cos_Ac_negg": [0.6272779107093811, 0.09554404020309448, 0.08841381222009659, 0.03544314578175545, 0.058252446353435516, 0.09822554141283035, 0.04386086389422417, 0.051703598350286484, 0.0914786159992218, 0.10188747197389603, 0.03786192834377289, 0.03893250972032547, 0.041348159313201904, 0.10928703099489212, 0.004410627298057079, 0.12595109641551971, 0.04410814866423607, 0.08702585101127625, 0.02068858966231346, 1.6868114471435547e-05, 0.03216884657740593, 0.02092152275145054, 0.029736410826444626, -0.01660814695060253, 0.10467110574245453, 0.07446694374084473, 0.03742540627717972, -0.04454159736633301, 0.0001198919489979744, -0.010586773976683617, 0.06511332094669342, 0.07579436898231506, 0.07500766962766647, 0.004093728959560394, 0.08108247816562653, 0.08558335900306702, 0.08037269115447998, 0.084686279296875, 0.06961989402770996, 0.10539840161800385, 0.030324822291731834, 0.08596324920654297, 0.13491132855415344, 0.1088460311293602, 0.09309998899698257, 0.035371311008930206, 0.07957285642623901, -0.0028438279405236244, -0.04044720530509949, 0.05673590302467346, 0.0667867511510849, 0.06647487729787827, 0.09840860962867737, 0.01284649409353733, 0.0485856756567955, 0.1468493640422821, 0.13607463240623474, 0.0837700366973877, 0.006812230218201876, 0.02709188312292099], "r_norm": [1.8533973693847656, 1.776321291923523, 1.8050003051757812, 1.6954245567321777, 1.7285454273223877, 1.7489041090011597, 1.7084928750991821, 1.684015154838562, 1.7439699172973633, 1.778260588645935, 1.7410922050476074, 1.7054730653762817, 1.7287631034851074, 1.654308557510376, 1.6868008375167847, 1.7702763080596924, 1.8197346925735474, 1.6785264015197754, 1.7118470668792725, 1.7954206466674805, 1.7487709522247314, 1.8190120458602905, 1.6632299423217773, 1.7829325199127197, 1.6722966432571411, 1.7235727310180664, 1.6544535160064697, 1.6766688823699951, 1.7740652561187744, 1.7941101789474487, 1.687652349472046, 1.6913474798202515, 1.754999041557312, 1.79796302318573, 1.637403964996338, 1.7292962074279785, 1.8579680919647217, 1.7303799390792847, 1.7899342775344849, 1.7514957189559937, 1.7011914253234863, 1.7407596111297607, 1.7763192653656006, 1.685621738433838, 1.7055363655090332, 1.734250545501709, 1.7574855089187622, 1.7126569747924805, 1.6533337831497192, 1.7627543210983276, 1.6665935516357422, 1.7230051755905151, 1.6893373727798462, 1.6761871576309204, 1.6779460906982422, 1.7561612129211426, 1.776987075805664, 1.7266807556152344, 1.7347924709320068, 1.6764953136444092]}, {"epoch_end": 1, "step": 782, "test_acc": 0.3938, "test_loss": 1.7509784790039062}, {"step": 1000, "epoch": 2, "train_loss": 1.7014912366867065, "cos_r_negg": [0.7236456274986267, 0.10970290005207062, 0.2043226659297943, 0.10859769582748413, 0.1280396282672882, 0.16228753328323364, 0.11362972855567932, 0.15102770924568176, 0.13198894262313843, 0.12518756091594696, 0.09627373516559601, 0.08484630286693573, 0.141693577170372, 0.10260644555091858, 0.05831114202737808, 0.171757310628891, 0.0899425745010376, 0.13327820599079132, 0.12464568763971329, 0.11597523838281631, 0.10539105534553528, 0.1261165738105774, 0.0746682733297348, 0.09788156300783157, 0.15284988284111023, 0.12805451452732086, 0.11427730321884155, 0.0011572642251849174, 0.07555311173200607, 0.06866436451673508, 0.17438331246376038, 0.16362999379634857, 0.12816017866134644, 0.08516138046979904, 0.16048109531402588, 0.17823109030723572, 0.12621425092220306, 0.16487407684326172, 0.1339254379272461, 0.14786984026432037, 0.06990502774715424, 0.18738672137260437, 0.13792017102241516, 0.19051727652549744, 0.16031207144260406, 0.10106445848941803, 0.18645861744880676, 0.13088129460811615, 0.05393320322036743, 0.16346518695354462, 0.16626185178756714, 0.13847103714942932, 0.14876221120357513, 0.08551368117332458, 0.09744732081890106, 0.1842239499092102, 0.16373170912265778, 0.15669608116149902, 0.06223825737833977, 0.10515368729829788], "cos_innovation_negg": [0.7236456274986267, 0.10970290005207062, 0.2043226659297943, 0.10859769582748413, 0.1280396282672882, 0.16228753328323364, 0.11362972855567932, 0.15102770924568176, 0.13198894262313843, 0.12518756091594696, 0.09627373516559601, 0.08484630286693573, 0.141693577170372, 0.10260644555091858, 0.05831114202737808, 0.171757310628891, 0.0899425745010376, 0.13327820599079132, 0.12464568763971329, 0.11597523838281631, 0.10539105534553528, 0.1261165738105774, 0.0746682733297348, 0.09788156300783157, 0.15284988284111023, 0.12805451452732086, 0.11427730321884155, 0.0011572642251849174, 0.07555311173200607, 0.06866436451673508, 0.17438331246376038, 0.16362999379634857, 0.12816017866134644, 0.08516138046979904, 0.16048109531402588, 0.17823109030723572, 0.12621425092220306, 0.16487407684326172, 0.1339254379272461, 0.14786984026432037, 0.06990502774715424, 0.18738672137260437, 0.13792017102241516, 0.19051727652549744, 0.16031207144260406, 0.10106445848941803, 0.18645861744880676, 0.13088129460811615, 0.05393320322036743, 0.16346518695354462, 0.16626185178756714, 0.13847103714942932, 0.14876221120357513, 0.08551368117332458, 0.09744732081890106, 0.1842239499092102, 0.16373170912265778, 0.15669608116149902, 0.06223825737833977, 0.10515368729829788], "cos_apical_negg": [0.7236456274986267, 0.10970290005207062, 0.2043226659297943, 0.10859769582748413, 0.1280396282672882, 0.16228753328323364, 0.11362972855567932, 0.15102770924568176, 0.13198894262313843, 0.12518756091594696, 0.09627373516559601, 0.08484630286693573, 0.141693577170372, 0.10260644555091858, 0.05831114202737808, 0.171757310628891, 0.0899425745010376, 0.13327820599079132, 0.12464568763971329, 0.11597523838281631, 0.10539105534553528, 0.1261165738105774, 0.0746682733297348, 0.09788156300783157, 0.15284988284111023, 0.12805451452732086, 0.11427730321884155, 0.0011572642251849174, 0.07555311173200607, 0.06866436451673508, 0.17438331246376038, 0.16362999379634857, 0.12816017866134644, 0.08516138046979904, 0.16048109531402588, 0.17823109030723572, 0.12621425092220306, 0.16487407684326172, 0.1339254379272461, 0.14786984026432037, 0.06990502774715424, 0.18738672137260437, 0.13792017102241516, 0.19051727652549744, 0.16031207144260406, 0.10106445848941803, 0.18645861744880676, 0.13088129460811615, 0.05393320322036743, 0.16346518695354462, 0.16626185178756714, 0.13847103714942932, 0.14876221120357513, 0.08551368117332458, 0.09744732081890106, 0.1842239499092102, 0.16373170912265778, 0.15669608116149902, 0.06223825737833977, 0.10515368729829788], "cos_Ac_negg": [0.7236456274986267, 0.10970290005207062, 0.2043226659297943, 0.10859769582748413, 0.1280396282672882, 0.16228753328323364, 0.11362972855567932, 0.15102770924568176, 0.13198894262313843, 0.12518756091594696, 0.09627373516559601, 0.08484630286693573, 0.141693577170372, 0.10260644555091858, 0.05831114202737808, 0.171757310628891, 0.0899425745010376, 0.13327820599079132, 0.12464568763971329, 0.11597523838281631, 0.10539105534553528, 0.1261165738105774, 0.0746682733297348, 0.09788156300783157, 0.15284988284111023, 0.12805451452732086, 0.11427730321884155, 0.0011572642251849174, 0.07555311173200607, 0.06866436451673508, 0.17438331246376038, 0.16362999379634857, 0.12816017866134644, 0.08516138046979904, 0.16048109531402588, 0.17823109030723572, 0.12621425092220306, 0.16487407684326172, 0.1339254379272461, 0.14786984026432037, 0.06990502774715424, 0.18738672137260437, 0.13792017102241516, 0.19051727652549744, 0.16031207144260406, 0.10106445848941803, 0.18645861744880676, 0.13088129460811615, 0.05393320322036743, 0.16346518695354462, 0.16626185178756714, 0.13847103714942932, 0.14876221120357513, 0.08551368117332458, 0.09744732081890106, 0.1842239499092102, 0.16373170912265778, 0.15669608116149902, 0.06223825737833977, 0.10515368729829788], "r_norm": [1.563770055770874, 1.4273457527160645, 1.4673240184783936, 1.3573946952819824, 1.3889734745025635, 1.4038348197937012, 1.3690142631530762, 1.3557652235031128, 1.4012200832366943, 1.4221785068511963, 1.3971067667007446, 1.364931583404541, 1.396085500717163, 1.3304088115692139, 1.357017993927002, 1.4245636463165283, 1.4541935920715332, 1.3431804180145264, 1.3720495700836182, 1.439864158630371, 1.4027150869369507, 1.4580072164535522, 1.332604169845581, 1.4325730800628662, 1.3468611240386963, 1.3816044330596924, 1.331197738647461, 1.3367949724197388, 1.4147512912750244, 1.4311048984527588, 1.3567042350769043, 1.3654303550720215, 1.4000955820083618, 1.4307591915130615, 1.3197827339172363, 1.3927544355392456, 1.4870507717132568, 1.3919496536254883, 1.4374040365219116, 1.4084552526474, 1.3586692810058594, 1.4012057781219482, 1.4319205284118652, 1.3604626655578613, 1.3759034872055054, 1.3878145217895508, 1.4185106754302979, 1.37095308303833, 1.3129594326019287, 1.4182674884796143, 1.339052438735962, 1.3837695121765137, 1.3614170551300049, 1.33738112449646, 1.3497247695922852, 1.419715404510498, 1.4297499656677246, 1.3911666870117188, 1.385725498199463, 1.3445806503295898]}, {"epoch_end": 2, "step": 1173, "test_acc": 0.3991, "test_loss": 1.7174166870117187}, {"step": 1500, "epoch": 3, "train_loss": 1.6487102508544922, "cos_r_negg": [0.7552416920661926, 0.20943275094032288, 0.261974036693573, 0.21944934129714966, 0.21169376373291016, 0.22628548741340637, 0.20678579807281494, 0.20754092931747437, 0.22920462489128113, 0.21733695268630981, 0.17192333936691284, 0.1506499946117401, 0.22385498881340027, 0.23543398082256317, 0.1682005226612091, 0.22371602058410645, 0.17566509544849396, 0.20814675092697144, 0.19247707724571228, 0.21146777272224426, 0.20350295305252075, 0.2292824536561966, 0.16761815547943115, 0.17521269619464874, 0.26212334632873535, 0.23650816082954407, 0.19538427889347076, 0.13976970314979553, 0.17279504239559174, 0.1606699526309967, 0.23262307047843933, 0.25370126962661743, 0.18869006633758545, 0.1556304693222046, 0.20108911395072937, 0.26171934604644775, 0.22476141154766083, 0.2629941701889038, 0.20963427424430847, 0.2220284342765808, 0.22897639870643616, 0.2647925615310669, 0.21590813994407654, 0.2863854169845581, 0.2832527458667755, 0.18859988451004028, 0.24929028749465942, 0.2249445915222168, 0.15656623244285583, 0.2601383626461029, 0.26267877221107483, 0.21730110049247742, 0.24681472778320312, 0.16060566902160645, 0.16291360557079315, 0.29399192333221436, 0.2724402844905853, 0.23936636745929718, 0.18187060952186584, 0.19209855794906616], "cos_innovation_negg": [0.7552416920661926, 0.20943275094032288, 0.261974036693573, 0.21944934129714966, 0.21169376373291016, 0.22628548741340637, 0.20678579807281494, 0.20754092931747437, 0.22920462489128113, 0.21733695268630981, 0.17192333936691284, 0.1506499946117401, 0.22385498881340027, 0.23543398082256317, 0.1682005226612091, 0.22371602058410645, 0.17566509544849396, 0.20814675092697144, 0.19247707724571228, 0.21146777272224426, 0.20350295305252075, 0.2292824536561966, 0.16761815547943115, 0.17521269619464874, 0.26212334632873535, 0.23650816082954407, 0.19538427889347076, 0.13976970314979553, 0.17279504239559174, 0.1606699526309967, 0.23262307047843933, 0.25370126962661743, 0.18869006633758545, 0.1556304693222046, 0.20108911395072937, 0.26171934604644775, 0.22476141154766083, 0.2629941701889038, 0.20963427424430847, 0.2220284342765808, 0.22897639870643616, 0.2647925615310669, 0.21590813994407654, 0.2863854169845581, 0.2832527458667755, 0.18859988451004028, 0.24929028749465942, 0.2249445915222168, 0.15656623244285583, 0.2601383626461029, 0.26267877221107483, 0.21730110049247742, 0.24681472778320312, 0.16060566902160645, 0.16291360557079315, 0.29399192333221436, 0.2724402844905853, 0.23936636745929718, 0.18187060952186584, 0.19209855794906616], "cos_apical_negg": [0.7552416920661926, 0.20943275094032288, 0.261974036693573, 0.21944934129714966, 0.21169376373291016, 0.22628548741340637, 0.20678579807281494, 0.20754092931747437, 0.22920462489128113, 0.21733695268630981, 0.17192333936691284, 0.1506499946117401, 0.22385498881340027, 0.23543398082256317, 0.1682005226612091, 0.22371602058410645, 0.17566509544849396, 0.20814675092697144, 0.19247707724571228, 0.21146777272224426, 0.20350295305252075, 0.2292824536561966, 0.16761815547943115, 0.17521269619464874, 0.26212334632873535, 0.23650816082954407, 0.19538427889347076, 0.13976970314979553, 0.17279504239559174, 0.1606699526309967, 0.23262307047843933, 0.25370126962661743, 0.18869006633758545, 0.1556304693222046, 0.20108911395072937, 0.26171934604644775, 0.22476141154766083, 0.2629941701889038, 0.20963427424430847, 0.2220284342765808, 0.22897639870643616, 0.2647925615310669, 0.21590813994407654, 0.2863854169845581, 0.2832527458667755, 0.18859988451004028, 0.24929028749465942, 0.2249445915222168, 0.15656623244285583, 0.2601383626461029, 0.26267877221107483, 0.21730110049247742, 0.24681472778320312, 0.16060566902160645, 0.16291360557079315, 0.29399192333221436, 0.2724402844905853, 0.23936636745929718, 0.18187060952186584, 0.19209855794906616], "cos_Ac_negg": [0.7552416920661926, 0.20943275094032288, 0.261974036693573, 0.21944934129714966, 0.21169376373291016, 0.22628548741340637, 0.20678579807281494, 0.20754092931747437, 0.22920462489128113, 0.21733695268630981, 0.17192333936691284, 0.1506499946117401, 0.22385498881340027, 0.23543398082256317, 0.1682005226612091, 0.22371602058410645, 0.17566509544849396, 0.20814675092697144, 0.19247707724571228, 0.21146777272224426, 0.20350295305252075, 0.2292824536561966, 0.16761815547943115, 0.17521269619464874, 0.26212334632873535, 0.23650816082954407, 0.19538427889347076, 0.13976970314979553, 0.17279504239559174, 0.1606699526309967, 0.23262307047843933, 0.25370126962661743, 0.18869006633758545, 0.1556304693222046, 0.20108911395072937, 0.26171934604644775, 0.22476141154766083, 0.2629941701889038, 0.20963427424430847, 0.2220284342765808, 0.22897639870643616, 0.2647925615310669, 0.21590813994407654, 0.2863854169845581, 0.2832527458667755, 0.18859988451004028, 0.24929028749465942, 0.2249445915222168, 0.15656623244285583, 0.2601383626461029, 0.26267877221107483, 0.21730110049247742, 0.24681472778320312, 0.16060566902160645, 0.16291360557079315, 0.29399192333221436, 0.2724402844905853, 0.23936636745929718, 0.18187060952186584, 0.19209855794906616], "r_norm": [1.3869777917861938, 1.1885013580322266, 1.2301267385482788, 1.1260701417922974, 1.1571848392486572, 1.1751539707183838, 1.147578239440918, 1.1430611610412598, 1.1667780876159668, 1.1825566291809082, 1.1609584093093872, 1.134331226348877, 1.1650300025939941, 1.1197609901428223, 1.1352510452270508, 1.1830880641937256, 1.2120609283447266, 1.123412013053894, 1.1413025856018066, 1.197468876838684, 1.1712942123413086, 1.2235493659973145, 1.1135492324829102, 1.1935064792633057, 1.1287078857421875, 1.1582484245300293, 1.115371823310852, 1.1097965240478516, 1.1770639419555664, 1.188248872756958, 1.1325643062591553, 1.1425156593322754, 1.1641089916229248, 1.1867830753326416, 1.107727289199829, 1.1736329793930054, 1.2492280006408691, 1.1619954109191895, 1.1984126567840576, 1.1784453392028809, 1.132127046585083, 1.1778287887573242, 1.1953659057617188, 1.1468782424926758, 1.1503671407699585, 1.1580755710601807, 1.1950898170471191, 1.1534717082977295, 1.0860962867736816, 1.1841540336608887, 1.1220223903656006, 1.1570713520050049, 1.1345725059509277, 1.1183111667633057, 1.12158203125, 1.1957414150238037, 1.1997321844100952, 1.1680374145507812, 1.1516621112823486, 1.1237547397613525]}, {"epoch_end": 3, "step": 1564, "test_acc": 0.3999, "test_loss": 1.7201217651367187}, {"epoch_end": 4, "step": 1955, "test_acc": 0.4119, "test_loss": 1.6888344360351562}], "final": {"test_acc": 0.4119, "test_loss": 1.6888344360351562, "wall_s": 89.61007714271545, "cos_r_negg": [0.7796429395675659, 0.3249361515045166, 0.3286120891571045, 0.3182772696018219, 0.2985309362411499, 0.31303828954696655, 0.3001897931098938, 0.2682933211326599, 0.31190255284309387, 0.28719842433929443, 0.27808183431625366, 0.2891491949558258, 0.3099251687526703, 0.29847848415374756, 0.2991320490837097, 0.3143173158168793, 0.2759777903556824, 0.30993515253067017, 0.2951984405517578, 0.28686046600341797, 0.28700345754623413, 0.30832070112228394, 0.266851544380188, 0.2890675663948059, 0.3486166000366211, 0.341863214969635, 0.29792240262031555, 0.2601832151412964, 0.25067198276519775, 0.279400110244751, 0.3517932593822479, 0.366024911403656, 0.3001241683959961, 0.2506081461906433, 0.3635994791984558, 0.36939728260040283, 0.3146013617515564, 0.36762088537216187, 0.32657390832901, 0.34473276138305664, 0.31394922733306885, 0.35495662689208984, 0.3203684687614441, 0.375538170337677, 0.3512468934059143, 0.28526657819747925, 0.3627834618091583, 0.3072934150695801, 0.2900983989238739, 0.36375075578689575, 0.37866610288619995, 0.34503334760665894, 0.3060999810695648, 0.282723605632782, 0.3261061906814575, 0.3777405619621277, 0.3757520318031311, 0.37055858969688416, 0.2731019854545593, 0.3126261830329895], "cos_innovation_negg": [0.7796429395675659, 0.3249361515045166, 0.3286120891571045, 0.3182772696018219, 0.2985309362411499, 0.31303828954696655, 0.3001897931098938, 0.2682933211326599, 0.31190255284309387, 0.28719842433929443, 0.27808183431625366, 0.2891491949558258, 0.3099251687526703, 0.29847848415374756, 0.2991320490837097, 0.3143173158168793, 0.2759777903556824, 0.30993515253067017, 0.2951984405517578, 0.28686046600341797, 0.28700345754623413, 0.30832070112228394, 0.266851544380188, 0.2890675663948059, 0.3486166000366211, 0.341863214969635, 0.29792240262031555, 0.2601832151412964, 0.25067198276519775, 0.279400110244751, 0.3517932593822479, 0.366024911403656, 0.3001241683959961, 0.2506081461906433, 0.3635994791984558, 0.36939728260040283, 0.3146013617515564, 0.36762088537216187, 0.32657390832901, 0.34473276138305664, 0.31394922733306885, 0.35495662689208984, 0.3203684687614441, 0.375538170337677, 0.3512468934059143, 0.28526657819747925, 0.3627834618091583, 0.3072934150695801, 0.2900983989238739, 0.36375075578689575, 0.37866610288619995, 0.34503334760665894, 0.3060999810695648, 0.282723605632782, 0.3261061906814575, 0.3777405619621277, 0.3757520318031311, 0.37055858969688416, 0.2731019854545593, 0.3126261830329895], "cos_apical_negg": [0.7796429395675659, 0.3249361515045166, 0.3286120891571045, 0.3182772696018219, 0.2985309362411499, 0.31303828954696655, 0.3001897931098938, 0.2682933211326599, 0.31190255284309387, 0.28719842433929443, 0.27808183431625366, 0.2891491949558258, 0.3099251687526703, 0.29847848415374756, 0.2991320490837097, 0.3143173158168793, 0.2759777903556824, 0.30993515253067017, 0.2951984405517578, 0.28686046600341797, 0.28700345754623413, 0.30832070112228394, 0.266851544380188, 0.2890675663948059, 0.3486166000366211, 0.341863214969635, 0.29792240262031555, 0.2601832151412964, 0.25067198276519775, 0.279400110244751, 0.3517932593822479, 0.366024911403656, 0.3001241683959961, 0.2506081461906433, 0.3635994791984558, 0.36939728260040283, 0.3146013617515564, 0.36762088537216187, 0.32657390832901, 0.34473276138305664, 0.31394922733306885, 0.35495662689208984, 0.3203684687614441, 0.375538170337677, 0.3512468934059143, 0.28526657819747925, 0.3627834618091583, 0.3072934150695801, 0.2900983989238739, 0.36375075578689575, 0.37866610288619995, 0.34503334760665894, 0.3060999810695648, 0.282723605632782, 0.3261061906814575, 0.3777405619621277, 0.3757520318031311, 0.37055858969688416, 0.2731019854545593, 0.3126261830329895], "cos_Ac_negg": [0.7796429395675659, 0.3249361515045166, 0.3286120891571045, 0.3182772696018219, 0.2985309362411499, 0.31303828954696655, 0.3001897931098938, 0.2682933211326599, 0.31190255284309387, 0.28719842433929443, 0.27808183431625366, 0.2891491949558258, 0.3099251687526703, 0.29847848415374756, 0.2991320490837097, 0.3143173158168793, 0.2759777903556824, 0.30993515253067017, 0.2951984405517578, 0.28686046600341797, 0.28700345754623413, 0.30832070112228394, 0.266851544380188, 0.2890675663948059, 0.3486166000366211, 0.341863214969635, 0.29792240262031555, 0.2601832151412964, 0.25067198276519775, 0.279400110244751, 0.3517932593822479, 0.366024911403656, 0.3001241683959961, 0.2506081461906433, 0.3635994791984558, 0.36939728260040283, 0.3146013617515564, 0.36762088537216187, 0.32657390832901, 0.34473276138305664, 0.31394922733306885, 0.35495662689208984, 0.3203684687614441, 0.375538170337677, 0.3512468934059143, 0.28526657819747925, 0.3627834618091583, 0.3072934150695801, 0.2900983989238739, 0.36375075578689575, 0.37866610288619995, 0.34503334760665894, 0.3060999810695648, 0.282723605632782, 0.3261061906814575, 0.3777405619621277, 0.3757520318031311, 0.37055858969688416, 0.2731019854545593, 0.3126261830329895]}} \ No newline at end of file -- cgit v1.2.3