results: reject diagonal SDIL on PickupLoc

1 files changed, 174 insertions, 0 deletions

diff --git a/results/babyai_shared/pickup_p2_analysis.json b/results/babyai_shared/pickup_p2_analysis.json
new file mode 100644
index 0000000..845e70c
--- /dev/null
+++ b/results/babyai_shared/pickup_p2_analysis.json
@@ -0,0 +1,174 @@
+{
+  "checks": {
+    "all_runs_finite": true,
+    "clean_kp_success_at_least_60_every_seed": true,
+    "raw_below_clean_kp_every_seed": true,
+    "sdil_above_raw_every_seed": false,
+    "sdil_nonzero_success_every_seed": false
+  },
+  "conditions": {
+    "bp": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 87.78380713263681,
+        "sample_std": 0.1078533752082175,
+        "values": [
+          87.8108042439459,
+          87.87559731108772,
+          87.66501984287682
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 49.333333333333336,
+        "sample_std": 1.1015141094572214,
+        "values": [
+          48.8,
+          50.6,
+          48.6
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 79.33333333333333,
+        "sample_std": 5.1588112325741635,
+        "values": [
+          80.80000000000001,
+          83.6,
+          73.6
+        ]
+      }
+    },
+    "clean_kp": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 86.43665127831322,
+        "sample_std": 0.3517610682148285,
+        "values": [
+          86.13428363165141,
+          86.8227099700332,
+          86.35296023325503
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 48.86666666666667,
+        "sample_std": 7.245228314783008,
+        "values": [
+          49.8,
+          55.60000000000001,
+          41.199999999999996
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 70.86666666666667,
+        "sample_std": 5.34540300944029,
+        "values": [
+          67.2,
+          77.0,
+          68.4
+        ]
+      }
+    },
+    "raw_shared": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 59.642018304041464,
+        "sample_std": 9.767595165741607,
+        "values": [
+          49.89066169919818,
+          69.42577144245566,
+          59.609621770470554
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 8.733333333333334,
+        "sample_std": 9.843441132720475,
+        "values": [
+          0.0,
+          19.400000000000002,
+          6.800000000000001
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 10.000000000000002,
+        "sample_std": 11.92644121270046,
+        "values": [
+          0.0,
+          23.200000000000003,
+          6.800000000000001
+        ]
+      }
+    },
+    "sdil": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 49.89066169919818,
+        "sample_std": 0.0,
+        "values": [
+          49.89066169919818,
+          49.89066169919818,
+          49.89066169919818
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 0.0,
+        "sample_std": 0.0,
+        "values": [
+          0.0,
+          0.0,
+          0.0
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 0.0,
+        "sample_std": 0.0,
+        "values": [
+          0.0,
+          0.0,
+          0.0
+        ]
+      }
+    }
+  },
+  "decision": "Close the diagonal per-cell predictor on PickupLoc. Raw shared feedback fails as hypothesized, but this SDIL implementation does not recover it. Screen the originally specified population predictor using neutral prediction only before another endpoint.",
+  "gate": "fail",
+  "paired_clean_kp_minus_raw_rollout_points": {
+    "mean": 60.866666666666674,
+    "sample_std": 6.730032194078519,
+    "values": [
+      67.2,
+      53.800000000000004,
+      61.60000000000001
+    ]
+  },
+  "paired_sdil_minus_raw_rollout_points": {
+    "mean": -10.000000000000002,
+    "sample_std": 11.92644121270046,
+    "values": [
+      0.0,
+      -23.200000000000003,
+      -6.800000000000001
+    ]
+  },
+  "sdil_predictor": {
+    "action_or_teaching_observations": 0,
+    "maximum_layer_residual_context_rms_ratio": {
+      "mean": 0.614041248957316,
+      "sample_std": 0.011718512673796736,
+      "values": [
+        0.6220114827156067,
+        0.6005861759185791,
+        0.6195260882377625
+      ]
+    },
+    "mean_per_cell_r2": {
+      "mean": 0.30999962737162906,
+      "sample_std": 0.005533015870761776,
+      "values": [
+        0.3146054223179817,
+        0.3038621172308922,
+        0.31153134256601334
+      ]
+    }
+  },
+  "stage": "babyai_pickup_p2_analysis",
+  "test_split_generated_or_read": false
+}