results: retain GoToObj as diagnostic only

1 files changed, 198 insertions, 0 deletions

diff --git a/results/babyai_shared/b1_analysis.json b/results/babyai_shared/b1_analysis.json
new file mode 100644
index 0000000..6c605a5
--- /dev/null
+++ b/results/babyai_shared/b1_analysis.json
@@ -0,0 +1,198 @@
+{
+  "checks": {
+    "all_runs_finite": true,
+    "bp_and_clean_kp_mean_rollout_at_least_80": true,
+    "mission_structurally_required": false,
+    "sdil_rollout_above_raw_every_seed": false
+  },
+  "conditions": {
+    "bp": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 99.74355321270836,
+        "sample_std": 0.0,
+        "values": [
+          99.74355321270836,
+          99.74355321270836,
+          99.74355321270836
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 88.2,
+        "sample_std": 14.208448191129108,
+        "values": [
+          71.8,
+          96.8,
+          96.0
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 97.8,
+        "sample_std": 0.0,
+        "values": [
+          97.8,
+          97.8,
+          97.8
+        ]
+      },
+      "training_wall_seconds": {
+        "mean": 145.09927169481912,
+        "sample_std": 56.693567058307366,
+        "values": [
+          178.87876749038696,
+          176.77257347106934,
+          79.6464741230011
+        ]
+      }
+    },
+    "clean_kp": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 99.74355321270836,
+        "sample_std": 0.0,
+        "values": [
+          99.74355321270836,
+          99.74355321270836,
+          99.74355321270836
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 78.46666666666667,
+        "sample_std": 15.37313674346694,
+        "values": [
+          60.8,
+          88.8,
+          85.8
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 97.8,
+        "sample_std": 0.0,
+        "values": [
+          97.8,
+          97.8,
+          97.8
+        ]
+      },
+      "training_wall_seconds": {
+        "mean": 150.14535276095071,
+        "sample_std": 56.34150377593018,
+        "values": [
+          182.5949375629425,
+          182.7532684803009,
+          85.08785223960876
+        ]
+      }
+    },
+    "raw_shared": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 93.80728498836491,
+        "sample_std": 6.19430813024119,
+        "values": [
+          87.24889585411027,
+          94.61461746687563,
+          99.55834164410885
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 88.8,
+        "sample_std": 17.528262891684385,
+        "values": [
+          68.60000000000001,
+          100.0,
+          97.8
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 83.93333333333334,
+        "sample_std": 15.297494348204001,
+        "values": [
+          67.2,
+          87.4,
+          97.2
+        ]
+      },
+      "training_wall_seconds": {
+        "mean": 147.5590082804362,
+        "sample_std": 54.54380970544501,
+        "values": [
+          178.62242436408997,
+          179.4754331111908,
+          84.57916736602783
+        ]
+      }
+    },
+    "sdil": {
+      "all_finite": true,
+      "expert_action_accuracy_percent": {
+        "mean": 97.56375552072944,
+        "sample_std": 2.472098159480726,
+        "values": [
+          99.4728593816783,
+          94.77133494799828,
+          98.44707223251176
+        ]
+      },
+      "mission_lesion_rollout_success_percent": {
+        "mean": 92.33333333333333,
+        "sample_std": 5.94754851458425,
+        "values": [
+          97.8,
+          86.0,
+          93.2
+        ]
+      },
+      "rollout_success_percent": {
+        "mean": 95.86666666666666,
+        "sample_std": 2.5324559884296756,
+        "values": [
+          97.8,
+          93.0,
+          96.8
+        ]
+      },
+      "training_wall_seconds": {
+        "mean": 150.52963081995645,
+        "sample_std": 56.5137490779585,
+        "values": [
+          183.09582328796387,
+          183.2198555469513,
+          85.27321362495422
+        ]
+      }
+    }
+  },
+  "decision": "Retain as an implementation diagnostic; do not use as the paper's task-required shared-feedback result. Move to PickupLoc.",
+  "gate": "fail",
+  "paired_sdil_minus_raw": {
+    "expert_action_accuracy_points": {
+      "mean": 3.7564705323645313,
+      "sample_std": 7.360419547002228,
+      "values": [
+        12.223963527568028,
+        0.15671748112265904,
+        -1.1112694115970934
+      ]
+    },
+    "rollout_success_points": {
+      "mean": 11.933333333333334,
+      "sample_std": 16.44181660685136,
+      "values": [
+        30.599999999999994,
+        5.600000000000005,
+        -0.40000000000000036
+      ]
+    }
+  },
+  "stage": "babyai_shared_b1_analysis",
+  "task_structure_audit": {
+    "audited_episode_seeds": 256,
+    "env_id": "BabyAI-GoToObjS6-v1",
+    "interpretation": "The sole object is the target, so the mission does not select among alternatives. Mission lesion is an input ablation, not evidence that language is required by the task.",
+    "maximum_task_object_count": 1,
+    "minimum_task_object_count": 1,
+    "mission_structurally_required_to_identify_the_only_object": false
+  },
+  "test_split_generated_or_read": false
+}