results: close shared-feedback feasibility screen

1 files changed, 345 insertions, 0 deletions

diff --git a/results/shared_feedback/s0.json b/results/shared_feedback/s0.json
new file mode 100644
index 0000000..487b8f5
--- /dev/null
+++ b/results/shared_feedback/s0.json
@@ -0,0 +1,345 @@
+{
+  "checks": {
+    "innovation_above_raw_by_5": false,
+    "innovation_within_2_of_exact_subtraction": true,
+    "innovation_within_3_of_oracle": true,
+    "matched_raw_below_innovation_by_3": false,
+    "nonzero_context_every_layer": true,
+    "oracle_at_least_90": true,
+    "oracle_context_lesion_drop_at_least_10": true,
+    "predictor_mean_r2_at_least_0p8": true,
+    "predictor_residual_ratio_at_most_0p25": true,
+    "raw_below_oracle_by_5_or_nonfinite": false,
+    "zero_instruction_observations": true
+  },
+  "config": {
+    "context_scale": 1.0,
+    "hidden_layers": 2,
+    "learning_rate": 0.03,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 64
+  },
+  "data": {
+    "batch_size": 128,
+    "data_seed": 3101,
+    "epochs": 40,
+    "neutral_examples_per_epoch": 512,
+    "test_generated": false,
+    "train_examples": 8192,
+    "validation_examples": 2048,
+    "validation_seed": 3102
+  },
+  "gate": "fail",
+  "provenance": {
+    "cuda_device_name": null,
+    "cuda_visible_devices": null,
+    "device": "cpu",
+    "git_commit": "b3457848820d0818840e7052c41f01c193d04a67",
+    "git_dirty_tracked": false,
+    "torch_version": "2.3.1+cu118"
+  },
+  "records": [
+    {
+      "condition": "oracle",
+      "context_lesion_validation": {
+        "accuracy": 0.76318359375,
+        "loss": 1.016611248254776
+      },
+      "epoch_train_loss": [
+        0.24325548473279923,
+        0.056528628687374294,
+        0.04177409497788176,
+        0.03513596647826489,
+        0.031240705080563203,
+        0.02832120239327196,
+        0.026365785437519662,
+        0.025093382872000802,
+        0.02279450709465891,
+        0.020808601460885257,
+        0.0205843910407566,
+        0.019572165729186963,
+        0.018927484703453956,
+        0.01822459184040781,
+        0.017170838073070627,
+        0.017234746352187358,
+        0.017531994708406273,
+        0.016618623849353753,
+        0.015849430204980308,
+        0.016377428259147564,
+        0.0150981526603573,
+        0.015522186793532455,
+        0.015784009628987405,
+        0.015121704243938439,
+        0.014048248613107717,
+        0.013343326936592348,
+        0.013460176676744595,
+        0.01443425157413003,
+        0.014310249827758525,
+        0.013833712246196228,
+        0.014911528505763272,
+        0.012738225999783026,
+        0.014052311023988295,
+        0.012899597548312158,
+        0.013465201343024091,
+        0.01266157620557351,
+        0.011880345572535589,
+        0.012348399211987271,
+        0.013484933984727832,
+        0.012156888736171823
+      ],
+      "epochs_completed": 40,
+      "finite": true,
+      "first_nonfinite_epoch": null,
+      "predictor": [
+        {
+          "context_rms": 0.6951092481613159,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.2886233329772949,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.44177088141441345
+        },
+        {
+          "context_rms": 0.6736197471618652,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.2944446802139282,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.5169849395751953
+        }
+      ],
+      "validation": {
+        "accuracy": 0.9931640625,
+        "loss": 0.015071565518155694
+      },
+      "wall_seconds": 1.3104579448699951
+    },
+    {
+      "condition": "raw_shared",
+      "context_lesion_validation": {
+        "accuracy": 0.7607421875,
+        "loss": 0.7937461286783218
+      },
+      "epoch_train_loss": [
+        0.3208125978708267,
+        0.1128524262458086,
+        0.09070953441550955,
+        0.08244586264481768,
+        0.0741222791839391,
+        0.0689597157179378,
+        0.06720075639896095,
+        0.05517955176765099,
+        0.04856250758166425,
+        0.04077540585421957,
+        0.03735822852468118,
+        0.03361609256535303,
+        0.03078002763504628,
+        0.02962772782484535,
+        0.026828612928511575,
+        0.028356117574730888,
+        0.026588890425045975,
+        0.024541094884625636,
+        0.023484145422116853,
+        0.02336278444272466,
+        0.023574378858029377,
+        0.025389555674337316,
+        0.02184313071484212,
+        0.020888249862764496,
+        0.021251495556498412,
+        0.021025175316026434,
+        0.02125847306160722,
+        0.025749154643563088,
+        0.026524975062784506,
+        0.02095157753501553,
+        0.025126684507995378,
+        0.01951841819754918,
+        0.024700971407582983,
+        0.021017702420067508,
+        0.019356194523425074,
+        0.027358750809071353,
+        0.024594824350060662,
+        0.022864215050503844,
+        0.02073163459135685,
+        0.01956268616777379
+      ],
+      "epochs_completed": 40,
+      "finite": true,
+      "first_nonfinite_epoch": null,
+      "predictor": [
+        {
+          "context_rms": 0.6951092481613159,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.8874340653419495,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.05456678196787834
+        },
+        {
+          "context_rms": 0.6736197471618652,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.5909687280654907,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.2560145854949951
+        }
+      ],
+      "validation": {
+        "accuracy": 0.990234375,
+        "loss": 0.021472798893228173
+      },
+      "wall_seconds": 1.2234578132629395
+    },
+    {
+      "condition": "innovation",
+      "context_lesion_validation": {
+        "accuracy": 0.76806640625,
+        "loss": 0.5023627653717995
+      },
+      "epoch_train_loss": [
+        0.3356692243833095,
+        0.11672432930208743,
+        0.0827551357797347,
+        0.07098419510293752,
+        0.061678838043008,
+        0.05678685213206336,
+        0.05060782533837482,
+        0.04663200181676075,
+        0.042038892686832696,
+        0.039127209020080045,
+        0.03786266638780944,
+        0.03385412292846013,
+        0.031796471928828396,
+        0.029597112981718965,
+        0.027298799192067236,
+        0.028002181876217946,
+        0.026437046893988736,
+        0.02457212092122063,
+        0.0231955681956606,
+        0.023024410373182036,
+        0.022563506630831398,
+        0.02402085693756817,
+        0.02065000399306882,
+        0.01983536635816563,
+        0.019318511425808538,
+        0.019305569570860825,
+        0.019423548750637565,
+        0.021987034422636498,
+        0.019796223194134654,
+        0.0184321171582269,
+        0.01941464086121414,
+        0.016116484581289114,
+        0.0172461594993365,
+        0.017044081127096433,
+        0.01634742740134243,
+        0.015952237394230906,
+        0.015391441356769064,
+        0.015357455056800973,
+        0.01551965332691907,
+        0.01406276357738534
+      ],
+      "epochs_completed": 40,
+      "finite": true,
+      "first_nonfinite_epoch": null,
+      "predictor": [
+        {
+          "context_rms": 0.6951092481613159,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.9216524958610535,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.01552559994161129
+        },
+        {
+          "context_rms": 0.6736197471618652,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.979547381401062,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.007725320290774107
+        }
+      ],
+      "validation": {
+        "accuracy": 0.9921875,
+        "loss": 0.01907800231128931
+      },
+      "wall_seconds": 1.2659540176391602
+    },
+    {
+      "condition": "matched_raw",
+      "context_lesion_validation": {
+        "accuracy": 0.7578125,
+        "loss": 1.1658207774162292
+      },
+      "epoch_train_loss": [
+        0.33571012574248016,
+        0.11975413211621344,
+        0.11104183469433337,
+        0.10416784440167248,
+        0.09184010204626247,
+        0.08320582384476438,
+        0.07883103750646114,
+        0.07580244168639183,
+        0.06573321402538568,
+        0.06347088766051456,
+        0.06262386264279485,
+        0.05813993178890087,
+        0.061674059164943174,
+        0.058328632556367666,
+        0.05570228290162049,
+        0.05565500335069373,
+        0.056083728617522866,
+        0.055472337000537664,
+        0.04814719475689344,
+        0.05002379676443525,
+        0.050801149918697774,
+        0.04792820388684049,
+        0.04658058172208257,
+        0.045018365170108154,
+        0.045646410377230495,
+        0.04163918021367863,
+        0.04184562063892372,
+        0.040560023349826224,
+        0.03966468955331948,
+        0.03746524166490417,
+        0.03624598232272547,
+        0.03414815798169002,
+        0.03465788248286117,
+        0.032305245971656404,
+        0.03320501872804016,
+        0.03220699344819877,
+        0.03228929900797084,
+        0.031194139643048402,
+        0.03019473489257507,
+        0.027403794330894016
+      ],
+      "epochs_completed": 40,
+      "finite": true,
+      "first_nonfinite_epoch": null,
+      "predictor": [
+        {
+          "context_rms": 0.6951092481613159,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.8674887418746948,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.061199601739645004
+        },
+        {
+          "context_rms": 0.6736197471618652,
+          "instruction_observations": 0,
+          "mean_per_cell_r2": 0.6172494888305664,
+          "neutral_observations": 512,
+          "residual_context_rms_ratio": 0.24446305632591248
+        }
+      ],
+      "validation": {
+        "accuracy": 0.99169921875,
+        "loss": 0.028344409074634314
+      },
+      "wall_seconds": 1.4177420139312744
+    }
+  ],
+  "stage": "shared_feedback_s0",
+  "summary": {
+    "innovation_validation_accuracy_percent": 99.21875,
+    "matched_raw_validation_accuracy_percent": 99.169921875,
+    "oracle_context_lesion_drop_points": 22.998046875,
+    "oracle_validation_accuracy_percent": 99.31640625,
+    "raw_validation_accuracy_percent": 99.0234375
+  }
+}