From 0d167497b75031fae67b71fa1e7ab12e57c761bb Mon Sep 17 00:00:00 2001
From: YurenHao0426 <Blackhao0426@gmail.com>
Date: Mon, 10 Aug 2026 10:50:16 -0500
Subject: results: close BabyAI population predictor rescue

---
 results/babyai_shared/population_p3_analysis.json | 176 ++++++++++++++++++++++
 1 file changed, 176 insertions(+)
 create mode 100644 results/babyai_shared/population_p3_analysis.json

(limited to 'results/babyai_shared/population_p3_analysis.json')

diff --git a/results/babyai_shared/population_p3_analysis.json b/results/babyai_shared/population_p3_analysis.json
new file mode 100644
index 0000000..08a9c7c
--- /dev/null
+++ b/results/babyai_shared/population_p3_analysis.json
@@ -0,0 +1,176 @@
+{
+  "checks": {
+    "all_seed_gates_pass": false,
+    "r2_at_least_0p8_every_seed_epoch_layer": false,
+    "residual_ratio_at_most_0p25_every_seed_epoch_layer": false,
+    "zero_action_or_teaching_observations": true
+  },
+  "decision": "Close the population-linear predictor rescue. It is highly predictive at initialization but fails the frozen early-layer threshold after clean task learning; do not run another PickupLoc downstream endpoint.",
+  "epoch_layer_means_across_seeds": [
+    {
+      "epoch": 0,
+      "layer": 0,
+      "mean_holdout_r2": 0.972669243812561,
+      "mean_holdout_residual_ratio": 0.10559078057607015
+    },
+    {
+      "epoch": 0,
+      "layer": 1,
+      "mean_holdout_r2": 0.9938111305236816,
+      "mean_holdout_residual_ratio": 0.05075520773728689
+    },
+    {
+      "epoch": 0,
+      "layer": 2,
+      "mean_holdout_r2": 0.9965593020121256,
+      "mean_holdout_residual_ratio": 0.037968567262093224
+    },
+    {
+      "epoch": 0,
+      "layer": 3,
+      "mean_holdout_r2": 0.9977092742919922,
+      "mean_holdout_residual_ratio": 0.03117166894177596
+    },
+    {
+      "epoch": 1,
+      "layer": 0,
+      "mean_holdout_r2": 0.8532736500104269,
+      "mean_holdout_residual_ratio": 0.24617203076680502
+    },
+    {
+      "epoch": 1,
+      "layer": 1,
+      "mean_holdout_r2": 0.9774375756581625,
+      "mean_holdout_residual_ratio": 0.09696057438850403
+    },
+    {
+      "epoch": 1,
+      "layer": 2,
+      "mean_holdout_r2": 0.9829351902008057,
+      "mean_holdout_residual_ratio": 0.08451603849728902
+    },
+    {
+      "epoch": 1,
+      "layer": 3,
+      "mean_holdout_r2": 0.9884803891181946,
+      "mean_holdout_residual_ratio": 0.07001137733459473
+    },
+    {
+      "epoch": 5,
+      "layer": 0,
+      "mean_holdout_r2": 0.7852290074030558,
+      "mean_holdout_residual_ratio": 0.2982979516188304
+    },
+    {
+      "epoch": 5,
+      "layer": 1,
+      "mean_holdout_r2": 0.9404525359471639,
+      "mean_holdout_residual_ratio": 0.15772825479507446
+    },
+    {
+      "epoch": 5,
+      "layer": 2,
+      "mean_holdout_r2": 0.9551665385564169,
+      "mean_holdout_residual_ratio": 0.13717531661192575
+    },
+    {
+      "epoch": 5,
+      "layer": 3,
+      "mean_holdout_r2": 0.9778080781300863,
+      "mean_holdout_residual_ratio": 0.09727375209331512
+    },
+    {
+      "epoch": 10,
+      "layer": 0,
+      "mean_holdout_r2": 0.7818694313367208,
+      "mean_holdout_residual_ratio": 0.30070162812868756
+    },
+    {
+      "epoch": 10,
+      "layer": 1,
+      "mean_holdout_r2": 0.9168808062871298,
+      "mean_holdout_residual_ratio": 0.1869215965270996
+    },
+    {
+      "epoch": 10,
+      "layer": 2,
+      "mean_holdout_r2": 0.9389026165008545,
+      "mean_holdout_residual_ratio": 0.16029799977938333
+    },
+    {
+      "epoch": 10,
+      "layer": 3,
+      "mean_holdout_r2": 0.9738152623176575,
+      "mean_holdout_residual_ratio": 0.10578999916712443
+    },
+    {
+      "epoch": 20,
+      "layer": 0,
+      "mean_holdout_r2": 0.8234108686447144,
+      "mean_holdout_residual_ratio": 0.2713727255662282
+    },
+    {
+      "epoch": 20,
+      "layer": 1,
+      "mean_holdout_r2": 0.9010587930679321,
+      "mean_holdout_residual_ratio": 0.20385040839513144
+    },
+    {
+      "epoch": 20,
+      "layer": 2,
+      "mean_holdout_r2": 0.9290995995203654,
+      "mean_holdout_residual_ratio": 0.17255698641141257
+    },
+    {
+      "epoch": 20,
+      "layer": 3,
+      "mean_holdout_r2": 0.9702359239260355,
+      "mean_holdout_residual_ratio": 0.11271155128876369
+    },
+    {
+      "epoch": 40,
+      "layer": 0,
+      "mean_holdout_r2": 0.7636886239051819,
+      "mean_holdout_residual_ratio": 0.31424906849861145
+    },
+    {
+      "epoch": 40,
+      "layer": 1,
+      "mean_holdout_r2": 0.8482151031494141,
+      "mean_holdout_residual_ratio": 0.2532327820857366
+    },
+    {
+      "epoch": 40,
+      "layer": 2,
+      "mean_holdout_r2": 0.8937068382898966,
+      "mean_holdout_residual_ratio": 0.21221035222212473
+    },
+    {
+      "epoch": 40,
+      "layer": 3,
+      "mean_holdout_r2": 0.9683659275372823,
+      "mean_holdout_residual_ratio": 0.11610875278711319
+    }
+  ],
+  "gate": "fail",
+  "maximum_holdout_residual_context_rms_ratio": {
+    "context_rms": 0.20538413524627686,
+    "epoch": 40,
+    "layer": 0,
+    "mean_per_cell_r2": 0.7517150640487671,
+    "observations": 4096,
+    "residual_context_rms_ratio": 0.32660192251205444,
+    "seed": 4103
+  },
+  "minimum_holdout_r2": {
+    "context_rms": 0.20538413524627686,
+    "epoch": 40,
+    "layer": 0,
+    "mean_per_cell_r2": 0.7517150640487671,
+    "observations": 4096,
+    "residual_context_rms_ratio": 0.32660192251205444,
+    "seed": 4103
+  },
+  "raw_sdil_rollout_or_test_outcomes_read": false,
+  "stage": "babyai_population_p3_analysis"
+}
-- 
cgit v1.2.3