results: close feedforward PickupLoc gate

1 files changed, 115 insertions, 0 deletions

diff --git a/results/babyai_shared/pickup_p0_analysis.json b/results/babyai_shared/pickup_p0_analysis.json
new file mode 100644
index 0000000..5c21b79
--- /dev/null
+++ b/results/babyai_shared/pickup_p0_analysis.json
@@ -0,0 +1,115 @@
+{
+  "best_clean_candidate_for_diagnosis": {
+    "bp_action_accuracy_percent": 84.81412488863691,
+    "bp_mission_lesion_drop_points": 52.0,
+    "bp_mission_lesion_success_percent": 11.600000000000001,
+    "bp_rollout_success_percent": 63.6,
+    "checks": {
+      "both_finite": true,
+      "bp_mission_lesion_drop_at_least_20": true,
+      "bp_success_at_least_70": false,
+      "clean_kp_success_at_least_60": false
+    },
+    "clean_kp_action_accuracy_percent": 83.84222888150968,
+    "clean_kp_rollout_success_percent": 52.2,
+    "eligible": false,
+    "hidden_layers": 4,
+    "learning_rate": 0.03,
+    "source_files": [
+      "results/babyai_shared/pickup_p0/d4_lr0.03_bp.json",
+      "results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json"
+    ]
+  },
+  "candidates": [
+    {
+      "bp_action_accuracy_percent": 81.9875273345752,
+      "bp_mission_lesion_drop_points": 34.199999999999996,
+      "bp_mission_lesion_success_percent": 15.6,
+      "bp_rollout_success_percent": 49.8,
+      "checks": {
+        "both_finite": true,
+        "bp_mission_lesion_drop_at_least_20": true,
+        "bp_success_at_least_70": false,
+        "clean_kp_success_at_least_60": false
+      },
+      "clean_kp_action_accuracy_percent": 80.30290758888799,
+      "clean_kp_rollout_success_percent": 45.6,
+      "eligible": false,
+      "hidden_layers": 2,
+      "learning_rate": 0.01,
+      "source_files": [
+        "results/babyai_shared/pickup_p0/d2_lr0.01_bp.json",
+        "results/babyai_shared/pickup_p0/d2_lr0.01_clean_kp.json"
+      ]
+    },
+    {
+      "bp_action_accuracy_percent": 79.03134364622986,
+      "bp_mission_lesion_drop_points": 31.4,
+      "bp_mission_lesion_success_percent": 13.0,
+      "bp_rollout_success_percent": 44.4,
+      "checks": {
+        "both_finite": true,
+        "bp_mission_lesion_drop_at_least_20": true,
+        "bp_success_at_least_70": false,
+        "clean_kp_success_at_least_60": false
+      },
+      "clean_kp_action_accuracy_percent": 80.88604519316434,
+      "clean_kp_rollout_success_percent": 47.0,
+      "eligible": false,
+      "hidden_layers": 2,
+      "learning_rate": 0.03,
+      "source_files": [
+        "results/babyai_shared/pickup_p0/d2_lr0.03_bp.json",
+        "results/babyai_shared/pickup_p0/d2_lr0.03_clean_kp.json"
+      ]
+    },
+    {
+      "bp_action_accuracy_percent": 84.88701708917145,
+      "bp_mission_lesion_drop_points": 51.199999999999996,
+      "bp_mission_lesion_success_percent": 11.600000000000001,
+      "bp_rollout_success_percent": 62.8,
+      "checks": {
+        "both_finite": true,
+        "bp_mission_lesion_drop_at_least_20": true,
+        "bp_success_at_least_70": false,
+        "clean_kp_success_at_least_60": false
+      },
+      "clean_kp_action_accuracy_percent": 81.9308334008261,
+      "clean_kp_rollout_success_percent": 51.0,
+      "eligible": false,
+      "hidden_layers": 4,
+      "learning_rate": 0.01,
+      "source_files": [
+        "results/babyai_shared/pickup_p0/d4_lr0.01_bp.json",
+        "results/babyai_shared/pickup_p0/d4_lr0.01_clean_kp.json"
+      ]
+    },
+    {
+      "bp_action_accuracy_percent": 84.81412488863691,
+      "bp_mission_lesion_drop_points": 52.0,
+      "bp_mission_lesion_success_percent": 11.600000000000001,
+      "bp_rollout_success_percent": 63.6,
+      "checks": {
+        "both_finite": true,
+        "bp_mission_lesion_drop_at_least_20": true,
+        "bp_success_at_least_70": false,
+        "clean_kp_success_at_least_60": false
+      },
+      "clean_kp_action_accuracy_percent": 83.84222888150968,
+      "clean_kp_rollout_success_percent": 52.2,
+      "eligible": false,
+      "hidden_layers": 4,
+      "learning_rate": 0.03,
+      "source_files": [
+        "results/babyai_shared/pickup_p0/d4_lr0.03_bp.json",
+        "results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json"
+      ]
+    }
+  ],
+  "decision": "Add task memory and repeat a clean-only gate; do not run raw or SDIL on the present feedforward policy.",
+  "gate": "fail",
+  "raw_or_sdil_results_run_or_read": false,
+  "selected": null,
+  "stage": "babyai_pickup_p0_clean_feasibility",
+  "test_split_generated_or_read": false
+}