results: close feedforward PickupLoc gate

1 files changed, 85 insertions, 0 deletions

diff --git a/experiments/analyze_babyai_pickup_p0.py b/experiments/analyze_babyai_pickup_p0.py
new file mode 100644
index 0000000..3b432ed
--- /dev/null
+++ b/experiments/analyze_babyai_pickup_p0.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Apply the frozen PickupLoc clean-feasibility gate."""
+
+import argparse
+import json
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_RESULTS = ROOT / "results" / "babyai_shared" / "pickup_p0"
+DEFAULT_OUT = ROOT / "results" / "babyai_shared" / "pickup_p0_analysis.json"
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
+    parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+    args = parser.parse_args()
+    candidates = []
+    for depth in (2, 4):
+        for learning_rate in (0.01, 0.03):
+            records = {}
+            sources = []
+            for condition in ("bp", "clean_kp"):
+                path = args.results / (
+                    f"d{depth}_lr{learning_rate}_{condition}.json")
+                with open(path, encoding="utf-8") as handle:
+                    records[condition] = json.load(handle)
+                sources.append(str(path.relative_to(ROOT)))
+            bp, kp = records["bp"], records["clean_kp"]
+            bp_success = 100.0 * bp["rollout"]["success"]
+            kp_success = 100.0 * kp["rollout"]["success"]
+            lesion_success = 100.0 * bp["mission_lesion_rollout"]["success"]
+            checks = {
+                "both_finite": bool(bp["finite"] and kp["finite"]),
+                "bp_success_at_least_70": bp_success >= 70.0,
+                "clean_kp_success_at_least_60": kp_success >= 60.0,
+                "bp_mission_lesion_drop_at_least_20": (
+                    bp_success - lesion_success >= 20.0),
+            }
+            candidates.append({
+                "hidden_layers": depth,
+                "learning_rate": learning_rate,
+                "bp_rollout_success_percent": bp_success,
+                "clean_kp_rollout_success_percent": kp_success,
+                "bp_mission_lesion_success_percent": lesion_success,
+                "bp_mission_lesion_drop_points": bp_success - lesion_success,
+                "bp_action_accuracy_percent": (
+                    100.0 * bp["validation"]["accuracy"]),
+                "clean_kp_action_accuracy_percent": (
+                    100.0 * kp["validation"]["accuracy"]),
+                "eligible": all(checks.values()),
+                "checks": checks,
+                "source_files": sources,
+            })
+    eligible = [row for row in candidates if row["eligible"]]
+    selected = max(eligible, key=lambda row: (
+        row["clean_kp_rollout_success_percent"],
+        row["clean_kp_action_accuracy_percent"],
+        -row["hidden_layers"], -row["learning_rate"])) if eligible else None
+    best_clean = max(candidates, key=lambda row: (
+        row["clean_kp_rollout_success_percent"],
+        row["clean_kp_action_accuracy_percent"]))
+    report = {
+        "stage": "babyai_pickup_p0_clean_feasibility",
+        "gate": "pass" if selected is not None else "fail",
+        "candidates": candidates,
+        "selected": selected,
+        "best_clean_candidate_for_diagnosis": best_clean,
+        "raw_or_sdil_results_run_or_read": False,
+        "test_split_generated_or_read": False,
+        "decision": (
+            "Add task memory and repeat a clean-only gate; do not run raw or "
+            "SDIL on the present feedforward policy."
+            if selected is None else "Open the frozen shared-feedback endpoint."),
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    with open(args.out, "w", encoding="utf-8") as handle:
+        json.dump(report, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(json.dumps(report, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+    main()