From bc93d2a673f5f971415715dad11c5e8baf2a6202 Mon Sep 17 00:00:00 2001
From: YurenHao0426 <Blackhao0426@gmail.com>
Date: Mon, 10 Aug 2026 10:45:41 -0500
Subject: results: reject diagonal SDIL on PickupLoc

---
 experiments/analyze_babyai_pickup_p2.py | 107 ++++++++++++++++++++++++++++++++
 1 file changed, 107 insertions(+)
 create mode 100644 experiments/analyze_babyai_pickup_p2.py

(limited to 'experiments/analyze_babyai_pickup_p2.py')

diff --git a/experiments/analyze_babyai_pickup_p2.py b/experiments/analyze_babyai_pickup_p2.py
new file mode 100644
index 0000000..5132ea9
--- /dev/null
+++ b/experiments/analyze_babyai_pickup_p2.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Summarize the frozen PickupLoc shared-feedback endpoint."""
+
+import argparse
+import json
+from pathlib import Path
+import statistics
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_RESULTS = ROOT / "results" / "babyai_shared" / "pickup_p2"
+DEFAULT_OUT = ROOT / "results" / "babyai_shared" / "pickup_p2_analysis.json"
+SEEDS = (4101, 4102, 4103)
+CONDITIONS = ("bp", "clean_kp", "raw_shared", "sdil")
+
+
+def summarize(values):
+    return {
+        "values": values,
+        "mean": statistics.mean(values),
+        "sample_std": statistics.stdev(values),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = parser.parse_args()
+    records = {}
+    for seed in SEEDS:
+        for condition in CONDITIONS:
+            path = args.results / f"seed{seed}_{condition}.json"
+            with open(path, encoding="utf-8") as handle:
+                records[(seed, condition)] = json.load(handle)
+
+    summaries = {}
+    for condition in CONDITIONS:
+        rows = [records[(seed, condition)] for seed in SEEDS]
+        summaries[condition] = {
+            "rollout_success_percent": summarize([
+                100.0 * row["rollout"]["success"] for row in rows]),
+            "expert_action_accuracy_percent": summarize([
+                100.0 * row["validation"]["accuracy"] for row in rows]),
+            "mission_lesion_rollout_success_percent": summarize([
+                100.0 * row["mission_lesion_rollout"]["success"]
+                for row in rows]),
+            "all_finite": all(row["finite"] for row in rows),
+        }
+    sdil_minus_raw = [100.0 * (
+        records[(seed, "sdil")]["rollout"]["success"]
+        - records[(seed, "raw_shared")]["rollout"]["success"])
+                      for seed in SEEDS]
+    clean_minus_raw = [100.0 * (
+        records[(seed, "clean_kp")]["rollout"]["success"]
+        - records[(seed, "raw_shared")]["rollout"]["success"])
+                       for seed in SEEDS]
+    predictor_r2 = [statistics.mean(
+        row["mean_per_cell_r2"]
+        for row in records[(seed, "sdil")]["predictor"])
+                    for seed in SEEDS]
+    residual_ratios = [max(
+        row["residual_context_rms_ratio"]
+        for row in records[(seed, "sdil")]["predictor"])
+                       for seed in SEEDS]
+    checks = {
+        "all_runs_finite": all(row["finite"] for row in records.values()),
+        "clean_kp_success_at_least_60_every_seed": all(
+            records[(seed, "clean_kp")]["rollout"]["success"] >= 0.6
+            for seed in SEEDS),
+        "raw_below_clean_kp_every_seed": all(
+            value > 0 for value in clean_minus_raw),
+        "sdil_above_raw_every_seed": all(
+            value > 0 for value in sdil_minus_raw),
+        "sdil_nonzero_success_every_seed": all(
+            records[(seed, "sdil")]["rollout"]["success"] > 0
+            for seed in SEEDS),
+    }
+    report = {
+        "stage": "babyai_pickup_p2_analysis",
+        "gate": "pass" if all(checks.values()) else "fail",
+        "checks": checks,
+        "conditions": summaries,
+        "paired_clean_kp_minus_raw_rollout_points": summarize(clean_minus_raw),
+        "paired_sdil_minus_raw_rollout_points": summarize(sdil_minus_raw),
+        "sdil_predictor": {
+            "mean_per_cell_r2": summarize(predictor_r2),
+            "maximum_layer_residual_context_rms_ratio": summarize(
+                residual_ratios),
+            "action_or_teaching_observations": 0,
+        },
+        "test_split_generated_or_read": False,
+        "decision": (
+            "Close the diagonal per-cell predictor on PickupLoc. Raw shared "
+            "feedback fails as hypothesized, but this SDIL implementation does "
+            "not recover it. Screen the originally specified population "
+            "predictor using neutral prediction only before another endpoint."),
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.out, "w", encoding="utf-8") as handle:
+        json.dump(report, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps(report, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()
-- 
cgit v1.2.3