results: close BabyAI population predictor rescue

4 files changed, 1721 insertions, 0 deletions

diff --git a/results/babyai_shared/population_p3/seed4101.json b/results/babyai_shared/population_p3/seed4101.json
new file mode 100644
index 0000000..75a0b0c
--- /dev/null
+++ b/results/babyai_shared/population_p3/seed4101.json
@@ -0,0 +1,515 @@
+{
+  "audits": [
+    {
+      "epoch": 0,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20984283089637756,
+            "mean_per_cell_r2": 0.9767748713493347,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09602414071559906
+          },
+          "holdout": {
+            "context_rms": 0.20962461829185486,
+            "mean_per_cell_r2": 0.9725972414016724,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1049511507153511
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2008315920829773,
+            "mean_per_cell_r2": 0.9949333667755127,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.04755431041121483
+          },
+          "holdout": {
+            "context_rms": 0.20081116259098053,
+            "mean_per_cell_r2": 0.9941949844360352,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.05143459141254425
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19726455211639404,
+            "mean_per_cell_r2": 0.997251570224762,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.03510858491063118
+          },
+          "holdout": {
+            "context_rms": 0.19765536487102509,
+            "mean_per_cell_r2": 0.996864914894104,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.03769101947546005
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19451536238193512,
+            "mean_per_cell_r2": 0.9981643557548523,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.029410211369395256
+          },
+          "holdout": {
+            "context_rms": 0.194533109664917,
+            "mean_per_cell_r2": 0.997877836227417,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.031882479786872864
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 1,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20984283089637756,
+            "mean_per_cell_r2": 0.8648810386657715,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.23362895846366882
+          },
+          "holdout": {
+            "context_rms": 0.20962461829185486,
+            "mean_per_cell_r2": 0.8518301844596863,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.24587911367416382
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2008315920829773,
+            "mean_per_cell_r2": 0.9841650128364563,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08432316780090332
+          },
+          "holdout": {
+            "context_rms": 0.20081116259098053,
+            "mean_per_cell_r2": 0.9819508790969849,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09081554412841797
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19726455211639404,
+            "mean_per_cell_r2": 0.9874575138092041,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.07510098814964294
+          },
+          "holdout": {
+            "context_rms": 0.19765536487102509,
+            "mean_per_cell_r2": 0.9855629205703735,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08102375268936157
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19451536238193512,
+            "mean_per_cell_r2": 0.9913246631622314,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.06404691189527512
+          },
+          "holdout": {
+            "context_rms": 0.194533109664917,
+            "mean_per_cell_r2": 0.9899281859397888,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.06954371184110641
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 5,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20984283089637756,
+            "mean_per_cell_r2": 0.8257500529289246,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2655547857284546
+          },
+          "holdout": {
+            "context_rms": 0.20962461829185486,
+            "mean_per_cell_r2": 0.800991415977478,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2851858139038086
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2008315920829773,
+            "mean_per_cell_r2": 0.9562175273895264,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.14005239307880402
+          },
+          "holdout": {
+            "context_rms": 0.20081116259098053,
+            "mean_per_cell_r2": 0.9500809907913208,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15084101259708405
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19726455211639404,
+            "mean_per_cell_r2": 0.9649862051010132,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.12556372582912445
+          },
+          "holdout": {
+            "context_rms": 0.19765536487102509,
+            "mean_per_cell_r2": 0.9601048827171326,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.13468508422374725
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19451536238193512,
+            "mean_per_cell_r2": 0.9821997880935669,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09148777276277542
+          },
+          "holdout": {
+            "context_rms": 0.194533109664917,
+            "mean_per_cell_r2": 0.9794628024101257,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09917064756155014
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 10,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20984283089637756,
+            "mean_per_cell_r2": 0.8274786472320557,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.26442500948905945
+          },
+          "holdout": {
+            "context_rms": 0.20962461829185486,
+            "mean_per_cell_r2": 0.8023961782455444,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.28441107273101807
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2008315920829773,
+            "mean_per_cell_r2": 0.9351796507835388,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.17075972259044647
+          },
+          "holdout": {
+            "context_rms": 0.20081116259098053,
+            "mean_per_cell_r2": 0.9255064725875854,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1848365217447281
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19726455211639404,
+            "mean_per_cell_r2": 0.9514586925506592,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1478380709886551
+          },
+          "holdout": {
+            "context_rms": 0.19765536487102509,
+            "mean_per_cell_r2": 0.943648099899292,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1601058840751648
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19451536238193512,
+            "mean_per_cell_r2": 0.9786931872367859,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10017189383506775
+          },
+          "holdout": {
+            "context_rms": 0.194533109664917,
+            "mean_per_cell_r2": 0.9755187630653381,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10835007578134537
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 20,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20984283089637756,
+            "mean_per_cell_r2": 0.8492339253425598,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.24697861075401306
+          },
+          "holdout": {
+            "context_rms": 0.20962461829185486,
+            "mean_per_cell_r2": 0.826335072517395,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2665298283100128
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2008315920829773,
+            "mean_per_cell_r2": 0.9172567129135132,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1933620125055313
+          },
+          "holdout": {
+            "context_rms": 0.20081116259098053,
+            "mean_per_cell_r2": 0.9072909951210022,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.206399068236351
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19726455211639404,
+            "mean_per_cell_r2": 0.9384824633598328,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1660328060388565
+          },
+          "holdout": {
+            "context_rms": 0.19765536487102509,
+            "mean_per_cell_r2": 0.9301103353500366,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1780754029750824
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19451536238193512,
+            "mean_per_cell_r2": 0.9720658659934998,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.11473160237073898
+          },
+          "holdout": {
+            "context_rms": 0.194533109664917,
+            "mean_per_cell_r2": 0.9683250784873962,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.12316417694091797
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 40,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20984283089637756,
+            "mean_per_cell_r2": 0.7983269691467285,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.28594422340393066
+          },
+          "holdout": {
+            "context_rms": 0.20962461829185486,
+            "mean_per_cell_r2": 0.7654898762702942,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.310098260641098
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2008315920829773,
+            "mean_per_cell_r2": 0.8684720993041992,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.24367430806159973
+          },
+          "holdout": {
+            "context_rms": 0.20081116259098053,
+            "mean_per_cell_r2": 0.851003885269165,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.26162275671958923
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19726455211639404,
+            "mean_per_cell_r2": 0.9062105417251587,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.20594285428524017
+          },
+          "holdout": {
+            "context_rms": 0.19765536487102509,
+            "mean_per_cell_r2": 0.8948484659194946,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2192971259355545
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.19451536238193512,
+            "mean_per_cell_r2": 0.9705915451049805,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.11770590394735336
+          },
+          "holdout": {
+            "context_rms": 0.194533109664917,
+            "mean_per_cell_r2": 0.9667372107505798,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.12624777853488922
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    }
+  ],
+  "calibration_examples": 4096,
+  "checks": {
+    "holdout_r2_at_least_0p8_every_epoch_and_layer": false,
+    "holdout_residual_ratio_at_most_0p25_every_epoch_and_layer": false,
+    "zero_action_or_teaching_observations": true
+  },
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 3964,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "epoch_clean_kp_train_loss": [
+    0.6793887242517973,
+    0.5070055065029546,
+    0.4780606180743167,
+    0.46464681211270786,
+    0.45651326932405173,
+    0.4414772361203244,
+    0.4295881901916705,
+    0.4249380529554267,
+    0.4178563620542225,
+    0.40975865175849513,
+    0.40546370631770084,
+    0.3982252176811821,
+    0.3858343799490678,
+    0.379007538933503,
+    0.3656208556576779,
+    0.35379995308424295,
+    0.33681938353337737,
+    0.32208908896697197,
+    0.30617407221543164,
+    0.2926546273419732,
+    0.28012341188757045,
+    0.27170656586948194,
+    0.2618562983211718,
+    0.25525567559819473,
+    0.24340780433855558,
+    0.241100468886526,
+    0.22936643666342685,
+    0.22153453873960596,
+    0.2213413413574821,
+    0.21846851916689622,
+    0.2104246342495868,
+    0.20888692896617087,
+    0.19663872825471979,
+    0.19428577112524134,
+    0.1947261790539089,
+    0.18938347912148426,
+    0.1853264942294673,
+    0.17872829263147555,
+    0.17811942146012658,
+    0.1693246652264344
+  ],
+  "gate": "fail",
+  "holdout_examples": 4096,
+  "model_seed": 4101,
+  "provenance": {
+    "cuda_visible_devices": "0",
+    "device": "cuda:0",
+    "git_commit": "b5eab6cd911e074ae9268303389dfbe52e636e26",
+    "torch_version": "2.3.1+cu118"
+  },
+  "raw_sdil_rollout_or_test_outcomes_read": false,
+  "ridge_relative_to_mean_soma_variance": 0.001,
+  "shuffle_seed": 4101,
+  "stage": "babyai_population_predictor_capacity_screen"
+}
diff --git a/results/babyai_shared/population_p3/seed4102.json b/results/babyai_shared/population_p3/seed4102.json
new file mode 100644
index 0000000..d6aeaf3
--- /dev/null
+++ b/results/babyai_shared/population_p3/seed4102.json
@@ -0,0 +1,515 @@
+{
+  "audits": [
+    {
+      "epoch": 0,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.975501537322998,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09886644780635834
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.9715493321418762,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1070699542760849
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9942841529846191,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.04786578193306923
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9934000372886658,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.05185777321457863
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.99674391746521,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.035994671285152435
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9962091445922852,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.03937750309705734
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9980149269104004,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.02863112837076187
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9977172613143921,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.031041312962770462
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 1,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8727436661720276,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2265353947877884
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.8598257303237915,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.23877772688865662
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9780939221382141,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09408465027809143
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9749774932861328,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10145710408687592
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9838603734970093,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08039840310811996
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9813672304153442,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08734050393104553
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9901689291000366,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.06410957127809525
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.988613486289978,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.06975055485963821
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 5,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.800828218460083,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2844180166721344
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.7751771807670593,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.3035990297794342
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9436342716217041,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15104201436042786
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9357898235321045,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.16277429461479187
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9581910967826843,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.12930455803871155
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9521858096122742,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1400468796491623
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9799500107765198,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09153742343187332
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9768326878547668,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.0993802398443222
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 10,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8033591508865356,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.282537043094635
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.7746613621711731,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.3040773570537567
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9248608350753784,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1742551028728485
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9133741855621338,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.18895503878593445
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9435411691665649,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15058007836341858
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.9352096319198608,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.16326111555099487
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9773434400558472,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09754791110754013
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9744316339492798,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10467227548360825
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 20,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8509368896484375,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.24803291261196136
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.8269829750061035,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2686232626438141
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.9160059690475464,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1840994507074356
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.9035534858703613,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.19918905198574066
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.9384135007858276,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1571311354637146
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.929930567741394,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.16975943744182587
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9770718216896057,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09782316535711288
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9735088348388672,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10616537183523178
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 40,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21022486686706543,
+            "mean_per_cell_r2": 0.8065019249916077,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2811703383922577
+          },
+          "holdout": {
+            "context_rms": 0.21084269881248474,
+            "mean_per_cell_r2": 0.7738609313964844,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.3060470223426819
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20877140760421753,
+            "mean_per_cell_r2": 0.87445068359375,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.22593224048614502
+          },
+          "holdout": {
+            "context_rms": 0.20807836949825287,
+            "mean_per_cell_r2": 0.8567341566085815,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2437417060136795
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20698049664497375,
+            "mean_per_cell_r2": 0.908510148525238,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.192082479596138
+          },
+          "holdout": {
+            "context_rms": 0.2065286487340927,
+            "mean_per_cell_r2": 0.8953220844268799,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.20811691880226135
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20679785311222076,
+            "mean_per_cell_r2": 0.9752291440963745,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10170674324035645
+          },
+          "holdout": {
+            "context_rms": 0.20624050498008728,
+            "mean_per_cell_r2": 0.9718630313873291,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10950509458780289
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    }
+  ],
+  "calibration_examples": 4096,
+  "checks": {
+    "holdout_r2_at_least_0p8_every_epoch_and_layer": false,
+    "holdout_residual_ratio_at_most_0p25_every_epoch_and_layer": false,
+    "zero_action_or_teaching_observations": true
+  },
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 3964,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "epoch_clean_kp_train_loss": [
+    0.681839566669966,
+    0.5106817564838811,
+    0.4866123088410026,
+    0.46724821322842647,
+    0.45275609010144285,
+    0.444956036241431,
+    0.4342330169050317,
+    0.42846722878907856,
+    0.41873464232996893,
+    0.4157650209100623,
+    0.40508943651851853,
+    0.3999078395492152,
+    0.3982298595654337,
+    0.3850154632016232,
+    0.3766156781347174,
+    0.368439452836388,
+    0.3616289800719211,
+    0.3508304416505914,
+    0.3401417228422667,
+    0.3305343171797301,
+    0.3165550116802517,
+    0.3019398627469414,
+    0.29349281496123264,
+    0.2876167286697187,
+    0.26772761492352737,
+    0.26045800096110294,
+    0.24848401973122045,
+    0.2440636997787576,
+    0.2351069336188467,
+    0.23047932257777767,
+    0.22535064524725865,
+    0.22159839981480647,
+    0.2125453116078126,
+    0.20711057270828045,
+    0.20062021220985213,
+    0.19858844374355517,
+    0.1931002575629636,
+    0.1893660755063358,
+    0.18631849710878573,
+    0.18104866024694943
+  ],
+  "gate": "fail",
+  "holdout_examples": 4096,
+  "model_seed": 4102,
+  "provenance": {
+    "cuda_visible_devices": "1",
+    "device": "cuda:0",
+    "git_commit": "b5eab6cd911e074ae9268303389dfbe52e636e26",
+    "torch_version": "2.3.1+cu118"
+  },
+  "raw_sdil_rollout_or_test_outcomes_read": false,
+  "ridge_relative_to_mean_soma_variance": 0.001,
+  "shuffle_seed": 4102,
+  "stage": "babyai_population_predictor_capacity_screen"
+}
diff --git a/results/babyai_shared/population_p3/seed4103.json b/results/babyai_shared/population_p3/seed4103.json
new file mode 100644
index 0000000..28ecab0
--- /dev/null
+++ b/results/babyai_shared/population_p3/seed4103.json
@@ -0,0 +1,515 @@
+{
+  "audits": [
+    {
+      "epoch": 0,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20582586526870728,
+            "mean_per_cell_r2": 0.9770644903182983,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09758477658033371
+          },
+          "holdout": {
+            "context_rms": 0.20538413524627686,
+            "mean_per_cell_r2": 0.9738611578941345,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10475123673677444
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2105173021554947,
+            "mean_per_cell_r2": 0.9947838187217712,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.04489493742585182
+          },
+          "holdout": {
+            "context_rms": 0.21071431040763855,
+            "mean_per_cell_r2": 0.993838369846344,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.04897325858473778
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2122984677553177,
+            "mean_per_cell_r2": 0.9970771074295044,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.03391746059060097
+          },
+          "holdout": {
+            "context_rms": 0.2119303047657013,
+            "mean_per_cell_r2": 0.9966038465499878,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.03683717921376228
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21755045652389526,
+            "mean_per_cell_r2": 0.997891902923584,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.028139039874076843
+          },
+          "holdout": {
+            "context_rms": 0.21835927665233612,
+            "mean_per_cell_r2": 0.9975327253341675,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.030591214075684547
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 1,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20582586526870728,
+            "mean_per_cell_r2": 0.8622075915336609,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.24054336547851562
+          },
+          "holdout": {
+            "context_rms": 0.20538413524627686,
+            "mean_per_cell_r2": 0.8481650352478027,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2538592517375946
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2105173021554947,
+            "mean_per_cell_r2": 0.9787577390670776,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09131480753421783
+          },
+          "holdout": {
+            "context_rms": 0.21071431040763855,
+            "mean_per_cell_r2": 0.9753843545913696,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.0986090749502182
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2122984677553177,
+            "mean_per_cell_r2": 0.9843738079071045,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.07851728051900864
+          },
+          "holdout": {
+            "context_rms": 0.2119303047657013,
+            "mean_per_cell_r2": 0.9818754196166992,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08518385887145996
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21755045652389526,
+            "mean_per_cell_r2": 0.9888011813163757,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.06514516472816467
+          },
+          "holdout": {
+            "context_rms": 0.21835927665233612,
+            "mean_per_cell_r2": 0.9868994951248169,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.07073986530303955
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 5,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20582586526870728,
+            "mean_per_cell_r2": 0.8066374063491821,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2849716544151306
+          },
+          "holdout": {
+            "context_rms": 0.20538413524627686,
+            "mean_per_cell_r2": 0.7795184254646301,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.3061090111732483
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2105173021554947,
+            "mean_per_cell_r2": 0.9446667432785034,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.14740806818008423
+          },
+          "holdout": {
+            "context_rms": 0.21071431040763855,
+            "mean_per_cell_r2": 0.9354867935180664,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15956945717334747
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2122984677553177,
+            "mean_per_cell_r2": 0.9587752819061279,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.12740013003349304
+          },
+          "holdout": {
+            "context_rms": 0.2119303047657013,
+            "mean_per_cell_r2": 0.9532089233398438,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.13679398596286774
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21755045652389526,
+            "mean_per_cell_r2": 0.979941725730896,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.08685733377933502
+          },
+          "holdout": {
+            "context_rms": 0.21835927665233612,
+            "mean_per_cell_r2": 0.9771287441253662,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.09327036887407303
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 10,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20582586526870728,
+            "mean_per_cell_r2": 0.7986479997634888,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.29081863164901733
+          },
+          "holdout": {
+            "context_rms": 0.20538413524627686,
+            "mean_per_cell_r2": 0.7685507535934448,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.31361645460128784
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2105173021554947,
+            "mean_per_cell_r2": 0.9244306087493896,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1723565012216568
+          },
+          "holdout": {
+            "context_rms": 0.21071431040763855,
+            "mean_per_cell_r2": 0.9117617607116699,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1869732290506363
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2122984677553177,
+            "mean_per_cell_r2": 0.9454227685928345,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.14660939574241638
+          },
+          "holdout": {
+            "context_rms": 0.2119303047657013,
+            "mean_per_cell_r2": 0.9378501176834106,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15752699971199036
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21755045652389526,
+            "mean_per_cell_r2": 0.9747556447982788,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.0976947769522667
+          },
+          "holdout": {
+            "context_rms": 0.21835927665233612,
+            "mean_per_cell_r2": 0.9714953899383545,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10434764623641968
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 20,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20582586526870728,
+            "mean_per_cell_r2": 0.842189610004425,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.25771480798721313
+          },
+          "holdout": {
+            "context_rms": 0.20538413524627686,
+            "mean_per_cell_r2": 0.8169145584106445,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2789650857448578
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2105173021554947,
+            "mean_per_cell_r2": 0.9072440266609192,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1907738745212555
+          },
+          "holdout": {
+            "context_rms": 0.21071431040763855,
+            "mean_per_cell_r2": 0.8923318982124329,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.2059631049633026
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2122984677553177,
+            "mean_per_cell_r2": 0.9364410638809204,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.15774686634540558
+          },
+          "holdout": {
+            "context_rms": 0.2119303047657013,
+            "mean_per_cell_r2": 0.9272578954696655,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1698361188173294
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21755045652389526,
+            "mean_per_cell_r2": 0.9725677967071533,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10174933075904846
+          },
+          "holdout": {
+            "context_rms": 0.21835927665233612,
+            "mean_per_cell_r2": 0.9688738584518433,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.1088051050901413
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    },
+    {
+      "epoch": 40,
+      "layers": [
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.20582586526870728,
+            "mean_per_cell_r2": 0.7883090376853943,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.29959437251091003
+          },
+          "holdout": {
+            "context_rms": 0.20538413524627686,
+            "mean_per_cell_r2": 0.7517150640487671,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.32660192251205444
+          },
+          "layer": 0,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2105173021554947,
+            "mean_per_cell_r2": 0.8573880195617676,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.23693645000457764
+          },
+          "holdout": {
+            "context_rms": 0.21071431040763855,
+            "mean_per_cell_r2": 0.8369072675704956,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.25433388352394104
+          },
+          "layer": 1,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.2122984677553177,
+            "mean_per_cell_r2": 0.9061275124549866,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.19269712269306183
+          },
+          "holdout": {
+            "context_rms": 0.2119303047657013,
+            "mean_per_cell_r2": 0.8909499645233154,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.20921701192855835
+          },
+          "layer": 2,
+          "teaching_observations": 0
+        },
+        {
+          "action_observations": 0,
+          "calibration": {
+            "context_rms": 0.21755045652389526,
+            "mean_per_cell_r2": 0.9699275493621826,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.10622629523277283
+          },
+          "holdout": {
+            "context_rms": 0.21835927665233612,
+            "mean_per_cell_r2": 0.966497540473938,
+            "observations": 4096,
+            "residual_context_rms_ratio": 0.11257338523864746
+          },
+          "layer": 3,
+          "teaching_observations": 0
+        }
+      ]
+    }
+  ],
+  "calibration_examples": 4096,
+  "checks": {
+    "holdout_r2_at_least_0p8_every_epoch_and_layer": false,
+    "holdout_residual_ratio_at_most_0p25_every_epoch_and_layer": false,
+    "zero_action_or_teaching_observations": true
+  },
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 3964,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "epoch_clean_kp_train_loss": [
+    0.6864033263608029,
+    0.5136532720766569,
+    0.4820757521453657,
+    0.4678087641063489,
+    0.4570458981237913,
+    0.44718988864045395,
+    0.43997702253492255,
+    0.42566254320897556,
+    0.4220644837931583,
+    0.41289369489017286,
+    0.405345781037682,
+    0.40259038736945707,
+    0.393158560112903,
+    0.38850406364390727,
+    0.3801098694926814,
+    0.36695603276553906,
+    0.35903412762441134,
+    0.34711836858799583,
+    0.3370369092727962,
+    0.3200993647387153,
+    0.3118796497897098,
+    0.293176792420839,
+    0.28278176423750423,
+    0.2673768097789664,
+    0.2630335626790398,
+    0.2525263265559548,
+    0.24664549178198764,
+    0.23551284742982764,
+    0.23284720298491027,
+    0.22212432547619468,
+    0.21526990802664506,
+    0.20761467494462665,
+    0.21185886748527225,
+    0.20015101647690722,
+    0.19496909729744258,
+    0.1936807933763454,
+    0.19314897574876483,
+    0.1823754488794427,
+    0.17861489517124077,
+    0.17446756489967044
+  ],
+  "gate": "fail",
+  "holdout_examples": 4096,
+  "model_seed": 4103,
+  "provenance": {
+    "cuda_visible_devices": "2",
+    "device": "cuda:0",
+    "git_commit": "b5eab6cd911e074ae9268303389dfbe52e636e26",
+    "torch_version": "2.3.1+cu118"
+  },
+  "raw_sdil_rollout_or_test_outcomes_read": false,
+  "ridge_relative_to_mean_soma_variance": 0.001,
+  "shuffle_seed": 4103,
+  "stage": "babyai_population_predictor_capacity_screen"
+}
diff --git a/results/babyai_shared/population_p3_analysis.json b/results/babyai_shared/population_p3_analysis.json
new file mode 100644
index 0000000..08a9c7c
--- /dev/null
+++ b/results/babyai_shared/population_p3_analysis.json
@@ -0,0 +1,176 @@
+{
+  "checks": {
+    "all_seed_gates_pass": false,
+    "r2_at_least_0p8_every_seed_epoch_layer": false,
+    "residual_ratio_at_most_0p25_every_seed_epoch_layer": false,
+    "zero_action_or_teaching_observations": true
+  },
+  "decision": "Close the population-linear predictor rescue. It is highly predictive at initialization but fails the frozen early-layer threshold after clean task learning; do not run another PickupLoc downstream endpoint.",
+  "epoch_layer_means_across_seeds": [
+    {
+      "epoch": 0,
+      "layer": 0,
+      "mean_holdout_r2": 0.972669243812561,
+      "mean_holdout_residual_ratio": 0.10559078057607015
+    },
+    {
+      "epoch": 0,
+      "layer": 1,
+      "mean_holdout_r2": 0.9938111305236816,
+      "mean_holdout_residual_ratio": 0.05075520773728689
+    },
+    {
+      "epoch": 0,
+      "layer": 2,
+      "mean_holdout_r2": 0.9965593020121256,
+      "mean_holdout_residual_ratio": 0.037968567262093224
+    },
+    {
+      "epoch": 0,
+      "layer": 3,
+      "mean_holdout_r2": 0.9977092742919922,
+      "mean_holdout_residual_ratio": 0.03117166894177596
+    },
+    {
+      "epoch": 1,
+      "layer": 0,
+      "mean_holdout_r2": 0.8532736500104269,
+      "mean_holdout_residual_ratio": 0.24617203076680502
+    },
+    {
+      "epoch": 1,
+      "layer": 1,
+      "mean_holdout_r2": 0.9774375756581625,
+      "mean_holdout_residual_ratio": 0.09696057438850403
+    },
+    {
+      "epoch": 1,
+      "layer": 2,
+      "mean_holdout_r2": 0.9829351902008057,
+      "mean_holdout_residual_ratio": 0.08451603849728902
+    },
+    {
+      "epoch": 1,
+      "layer": 3,
+      "mean_holdout_r2": 0.9884803891181946,
+      "mean_holdout_residual_ratio": 0.07001137733459473
+    },
+    {
+      "epoch": 5,
+      "layer": 0,
+      "mean_holdout_r2": 0.7852290074030558,
+      "mean_holdout_residual_ratio": 0.2982979516188304
+    },
+    {
+      "epoch": 5,
+      "layer": 1,
+      "mean_holdout_r2": 0.9404525359471639,
+      "mean_holdout_residual_ratio": 0.15772825479507446
+    },
+    {
+      "epoch": 5,
+      "layer": 2,
+      "mean_holdout_r2": 0.9551665385564169,
+      "mean_holdout_residual_ratio": 0.13717531661192575
+    },
+    {
+      "epoch": 5,
+      "layer": 3,
+      "mean_holdout_r2": 0.9778080781300863,
+      "mean_holdout_residual_ratio": 0.09727375209331512
+    },
+    {
+      "epoch": 10,
+      "layer": 0,
+      "mean_holdout_r2": 0.7818694313367208,
+      "mean_holdout_residual_ratio": 0.30070162812868756
+    },
+    {
+      "epoch": 10,
+      "layer": 1,
+      "mean_holdout_r2": 0.9168808062871298,
+      "mean_holdout_residual_ratio": 0.1869215965270996
+    },
+    {
+      "epoch": 10,
+      "layer": 2,
+      "mean_holdout_r2": 0.9389026165008545,
+      "mean_holdout_residual_ratio": 0.16029799977938333
+    },
+    {
+      "epoch": 10,
+      "layer": 3,
+      "mean_holdout_r2": 0.9738152623176575,
+      "mean_holdout_residual_ratio": 0.10578999916712443
+    },
+    {
+      "epoch": 20,
+      "layer": 0,
+      "mean_holdout_r2": 0.8234108686447144,
+      "mean_holdout_residual_ratio": 0.2713727255662282
+    },
+    {
+      "epoch": 20,
+      "layer": 1,
+      "mean_holdout_r2": 0.9010587930679321,
+      "mean_holdout_residual_ratio": 0.20385040839513144
+    },
+    {
+      "epoch": 20,
+      "layer": 2,
+      "mean_holdout_r2": 0.9290995995203654,
+      "mean_holdout_residual_ratio": 0.17255698641141257
+    },
+    {
+      "epoch": 20,
+      "layer": 3,
+      "mean_holdout_r2": 0.9702359239260355,
+      "mean_holdout_residual_ratio": 0.11271155128876369
+    },
+    {
+      "epoch": 40,
+      "layer": 0,
+      "mean_holdout_r2": 0.7636886239051819,
+      "mean_holdout_residual_ratio": 0.31424906849861145
+    },
+    {
+      "epoch": 40,
+      "layer": 1,
+      "mean_holdout_r2": 0.8482151031494141,
+      "mean_holdout_residual_ratio": 0.2532327820857366
+    },
+    {
+      "epoch": 40,
+      "layer": 2,
+      "mean_holdout_r2": 0.8937068382898966,
+      "mean_holdout_residual_ratio": 0.21221035222212473
+    },
+    {
+      "epoch": 40,
+      "layer": 3,
+      "mean_holdout_r2": 0.9683659275372823,
+      "mean_holdout_residual_ratio": 0.11610875278711319
+    }
+  ],
+  "gate": "fail",
+  "maximum_holdout_residual_context_rms_ratio": {
+    "context_rms": 0.20538413524627686,
+    "epoch": 40,
+    "layer": 0,
+    "mean_per_cell_r2": 0.7517150640487671,
+    "observations": 4096,
+    "residual_context_rms_ratio": 0.32660192251205444,
+    "seed": 4103
+  },
+  "minimum_holdout_r2": {
+    "context_rms": 0.20538413524627686,
+    "epoch": 40,
+    "layer": 0,
+    "mean_per_cell_r2": 0.7517150640487671,
+    "observations": 4096,
+    "residual_context_rms_ratio": 0.32660192251205444,
+    "seed": 4103
+  },
+  "raw_sdil_rollout_or_test_outcomes_read": false,
+  "stage": "babyai_population_p3_analysis"
+}