summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--BABYAI_SHARED_FEEDBACK.md16
-rw-r--r--experiments/analyze_babyai_pickup_p0.py85
-rw-r--r--results/babyai_shared/pickup_p0/d2_lr0.01_bp.json161
-rw-r--r--results/babyai_shared/pickup_p0/d2_lr0.01_clean_kp.json161
-rw-r--r--results/babyai_shared/pickup_p0/d2_lr0.03_bp.json161
-rw-r--r--results/babyai_shared/pickup_p0/d2_lr0.03_clean_kp.json161
-rw-r--r--results/babyai_shared/pickup_p0/d4_lr0.01_bp.json161
-rw-r--r--results/babyai_shared/pickup_p0/d4_lr0.01_clean_kp.json161
-rw-r--r--results/babyai_shared/pickup_p0/d4_lr0.03_bp.json161
-rw-r--r--results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json161
-rw-r--r--results/babyai_shared/pickup_p0_analysis.json115
11 files changed, 1504 insertions, 0 deletions
diff --git a/BABYAI_SHARED_FEEDBACK.md b/BABYAI_SHARED_FEEDBACK.md
index e214ead..9443253 100644
--- a/BABYAI_SHARED_FEEDBACK.md
+++ b/BABYAI_SHARED_FEEDBACK.md
@@ -168,3 +168,19 @@ If no candidate is eligible, raw and SDIL are not run on this architecture;
the next method change must supply task memory rather than tune shared-feedback
strength. If P0 passes, the chosen configuration enters the same 40-epoch,
three-seed BP/clean-KP/raw/SDIL comparison as B1.
+
+## P0 outcome
+
+All eight runs were finite, but no feedforward candidate passed. The best BP
+rollout success was `63.6%` (four layers, learning rate `0.03`) and the best
+clean-KP success was `52.2%` at the same setting, below the frozen `70%` and
+`60%` thresholds. Their expert-action accuracies were `84.81%` and `83.84%`.
+Removing the mission reduced the best BP policy to `11.6%` success, confirming
+that this task depends on the language signal. Raw and SDIL were not run.
+
+The next clean-only feasibility stage keeps the best P0 depth and learning rate
+and adds a fixed basal history of observations and previous actions. It tests
+history lengths four and eight for 25 epochs, using only BP and clean KP. The
+same `70%` BP, `60%` clean-KP, and 20-point BP mission-lesion thresholds apply;
+ties favor the shorter history. This changes the missing state information,
+not the shared-feedback signal or its strength.
diff --git a/experiments/analyze_babyai_pickup_p0.py b/experiments/analyze_babyai_pickup_p0.py
new file mode 100644
index 0000000..3b432ed
--- /dev/null
+++ b/experiments/analyze_babyai_pickup_p0.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Apply the frozen PickupLoc clean-feasibility gate."""
+
+import argparse
+import json
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_RESULTS = ROOT / "results" / "babyai_shared" / "pickup_p0"
+DEFAULT_OUT = ROOT / "results" / "babyai_shared" / "pickup_p0_analysis.json"
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
+ parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+ args = parser.parse_args()
+ candidates = []
+ for depth in (2, 4):
+ for learning_rate in (0.01, 0.03):
+ records = {}
+ sources = []
+ for condition in ("bp", "clean_kp"):
+ path = args.results / (
+ f"d{depth}_lr{learning_rate}_{condition}.json")
+ with open(path, encoding="utf-8") as handle:
+ records[condition] = json.load(handle)
+ sources.append(str(path.relative_to(ROOT)))
+ bp, kp = records["bp"], records["clean_kp"]
+ bp_success = 100.0 * bp["rollout"]["success"]
+ kp_success = 100.0 * kp["rollout"]["success"]
+ lesion_success = 100.0 * bp["mission_lesion_rollout"]["success"]
+ checks = {
+ "both_finite": bool(bp["finite"] and kp["finite"]),
+ "bp_success_at_least_70": bp_success >= 70.0,
+ "clean_kp_success_at_least_60": kp_success >= 60.0,
+ "bp_mission_lesion_drop_at_least_20": (
+ bp_success - lesion_success >= 20.0),
+ }
+ candidates.append({
+ "hidden_layers": depth,
+ "learning_rate": learning_rate,
+ "bp_rollout_success_percent": bp_success,
+ "clean_kp_rollout_success_percent": kp_success,
+ "bp_mission_lesion_success_percent": lesion_success,
+ "bp_mission_lesion_drop_points": bp_success - lesion_success,
+ "bp_action_accuracy_percent": (
+ 100.0 * bp["validation"]["accuracy"]),
+ "clean_kp_action_accuracy_percent": (
+ 100.0 * kp["validation"]["accuracy"]),
+ "eligible": all(checks.values()),
+ "checks": checks,
+ "source_files": sources,
+ })
+ eligible = [row for row in candidates if row["eligible"]]
+ selected = max(eligible, key=lambda row: (
+ row["clean_kp_rollout_success_percent"],
+ row["clean_kp_action_accuracy_percent"],
+ -row["hidden_layers"], -row["learning_rate"])) if eligible else None
+ best_clean = max(candidates, key=lambda row: (
+ row["clean_kp_rollout_success_percent"],
+ row["clean_kp_action_accuracy_percent"]))
+ report = {
+ "stage": "babyai_pickup_p0_clean_feasibility",
+ "gate": "pass" if selected is not None else "fail",
+ "candidates": candidates,
+ "selected": selected,
+ "best_clean_candidate_for_diagnosis": best_clean,
+ "raw_or_sdil_results_run_or_read": False,
+ "test_split_generated_or_read": False,
+ "decision": (
+ "Add task memory and repeat a clean-only gate; do not run raw or "
+ "SDIL on the present feedforward policy."
+ if selected is None else "Open the frozen shared-feedback endpoint."),
+ }
+ args.out.parent.mkdir(parents=True, exist_ok=True)
+ with open(args.out, "w", encoding="utf-8") as handle:
+ json.dump(report, handle, indent=2, sort_keys=True)
+ handle.write("\n")
+ print(json.dumps(report, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/results/babyai_shared/pickup_p0/d2_lr0.01_bp.json b/results/babyai_shared/pickup_p0/d2_lr0.01_bp.json
new file mode 100644
index 0000000..c7f82a4
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d2_lr0.01_bp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7629744564859491
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6666856745669716
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.6544169373261302
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.647978503955038
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.6390260830678438
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.6251882618351986
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.6098556981588665
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.5832556275317543
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.5497130578442624
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.5191664356934397
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.4923379595656144
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.47387167209073117
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.4634372958384062
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.4495846606555738
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.4385373055934906
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 55.368,
+ "mean_return": 0.13698749999999998,
+ "success": 0.156
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.6326233093059043,
+ "loss": 0.8153243552646898
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 41183232,
+ "cuda_visible_devices": "0",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 35.04,
+ "mean_return": 0.45705000000000007,
+ "success": 0.498
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 399879,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 129.50609016418457,
+ "training_wall_seconds": 113.70090627670288
+ },
+ "validation": {
+ "accuracy": 0.819875273345752,
+ "loss": 0.45946065495349603
+ }
+}
diff --git a/results/babyai_shared/pickup_p0/d2_lr0.01_clean_kp.json b/results/babyai_shared/pickup_p0/d2_lr0.01_clean_kp.json
new file mode 100644
index 0000000..d0b4bfb
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d2_lr0.01_clean_kp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.8030404388277155
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6701020826791462
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.6569775147187082
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.6511933691878068
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.6437298444697731
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.6332571965769718
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.6275139563962033
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.6194480020121524
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.6092520252654426
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.5939579413439098
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.5717180889531186
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.5435944906033968
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.5202985606695476
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.4997128319112878
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.48156642286401047
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 57.592,
+ "mean_return": 0.10091249999999999,
+ "success": 0.108
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.5193164331416539,
+ "loss": 0.9636234503733906
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 41183232,
+ "cuda_visible_devices": "1",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 37.75,
+ "mean_return": 0.41474062500000003,
+ "success": 0.456
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 399879,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 130.66909670829773,
+ "training_wall_seconds": 114.85607838630676
+ },
+ "validation": {
+ "accuracy": 0.8030290758888798,
+ "loss": 0.49055810259297433
+ }
+}
diff --git a/results/babyai_shared/pickup_p0/d2_lr0.03_bp.json b/results/babyai_shared/pickup_p0/d2_lr0.03_bp.json
new file mode 100644
index 0000000..f9f04ae
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d2_lr0.03_bp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7350854834757353
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6576031963448775
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.6314827724506981
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.5810897907457854
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.5212368984598863
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.4878401596295206
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.4724069481146963
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.44722843628180653
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.4356277753804859
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.4145806948762191
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.4113378077431729
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.39819210579520775
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.39019120831238596
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.3844031980163173
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.3826121056079865
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 56.53,
+ "mean_return": 0.118046875,
+ "success": 0.13
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.5643476148052159,
+ "loss": 1.0549333398908103
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 41183232,
+ "cuda_visible_devices": "2",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 38.216,
+ "mean_return": 0.40698749999999995,
+ "success": 0.444
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 399879,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 130.98018527030945,
+ "training_wall_seconds": 115.7611768245697
+ },
+ "validation": {
+ "accuracy": 0.7903134364622986,
+ "loss": 0.5078755087762494
+ }
+}
diff --git a/results/babyai_shared/pickup_p0/d2_lr0.03_clean_kp.json b/results/babyai_shared/pickup_p0/d2_lr0.03_clean_kp.json
new file mode 100644
index 0000000..202560b
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d2_lr0.03_clean_kp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 2,
+ "input_dim": 984,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7569978703950581
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6640346079123648
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.6450717353820801
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.6339595061854312
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.615129813269565
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.5775934231908698
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.5496430050699335
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.5214913842552586
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.5074587830744292
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.489895092625367
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.4762643372385125
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.46682736503450495
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.4581571973624982
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.4478275369343005
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.4360358424563157
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 55.146,
+ "mean_return": 0.13970937500000002,
+ "success": 0.152
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.5446667206608893,
+ "loss": 1.0876726227906528
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 41183232,
+ "cuda_visible_devices": "3",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 36.52,
+ "mean_return": 0.4334375,
+ "success": 0.47
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 399879,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 131.08930659294128,
+ "training_wall_seconds": 116.1480062007904
+ },
+ "validation": {
+ "accuracy": 0.8088604519316434,
+ "loss": 0.476227336193643
+ }
+}
diff --git a/results/babyai_shared/pickup_p0/d4_lr0.01_bp.json b/results/babyai_shared/pickup_p0/d4_lr0.01_bp.json
new file mode 100644
index 0000000..8b01dcc
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d4_lr0.01_bp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7549157315806339
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6568164462792246
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.6382872536307888
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.6060633628619344
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.5384791112573524
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.49386912408627964
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.47081105527124906
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.44292464864881415
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.42659055452597766
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.4093763682716771
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.3986523394208205
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.38867081842924417
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.3751908119728691
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.3708347668145832
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.3584983548365141
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 57.92,
+ "mean_return": 0.09709999999999999,
+ "success": 0.116
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.6245241759131773,
+ "loss": 1.0232753310308211
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 44869632,
+ "cuda_visible_devices": "4",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 27.928,
+ "mean_return": 0.5700625,
+ "success": 0.628
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 675335,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 131.81658458709717,
+ "training_wall_seconds": 119.09631705284119
+ },
+ "validation": {
+ "accuracy": 0.8488701708917146,
+ "loss": 0.3772129754918958
+ }
+}
diff --git a/results/babyai_shared/pickup_p0/d4_lr0.01_clean_kp.json b/results/babyai_shared/pickup_p0/d4_lr0.01_clean_kp.json
new file mode 100644
index 0000000..905f687
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d4_lr0.01_clean_kp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 984,
+ "learning_rate": 0.01,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.01,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.8088110878593043
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6625812041132073
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.6475289752608852
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.6388770164941486
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.6301077225961184
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.6097575246660333
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.5780363850844534
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.5507564557853498
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.5344218108528539
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.5059270592739707
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.48786478500617175
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.4668995203469929
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.4487882217608
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.4371816935664729
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.4205812331249839
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 54.618,
+ "mean_return": 0.14853437500000002,
+ "success": 0.166
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.6582975621608488,
+ "loss": 0.9520026964147612
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 44869632,
+ "cuda_visible_devices": "5",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 34.494,
+ "mean_return": 0.465928125,
+ "success": 0.51
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 675335,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 132.5482611656189,
+ "training_wall_seconds": 119.81917142868042
+ },
+ "validation": {
+ "accuracy": 0.8193083340082611,
+ "loss": 0.4542945731436375
+ }
+}
diff --git a/results/babyai_shared/pickup_p0/d4_lr0.03_bp.json b/results/babyai_shared/pickup_p0/d4_lr0.03_bp.json
new file mode 100644
index 0000000..397fb5a
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d4_lr0.03_bp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 984,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7262089006524337
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6357596055457466
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.5492566361552791
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.49844952420184485
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.4580263258281507
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.425203683439054
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.412210748948549
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.3926050000441702
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.3829512075374001
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.37295488219512135
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.3644254217649761
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.36054437838102643
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.34705134504719787
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.3469698649017434
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.3360688839774383
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 57.588,
+ "mean_return": 0.10176874999999999,
+ "success": 0.116
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.5888879889851786,
+ "loss": 1.111792022712493
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 44869632,
+ "cuda_visible_devices": "6",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 27.536,
+ "mean_return": 0.576375,
+ "success": 0.636
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 675335,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 131.90459489822388,
+ "training_wall_seconds": 118.93379855155945
+ },
+ "validation": {
+ "accuracy": 0.8481412488863691,
+ "loss": 0.38257111094978313
+ }
+}
diff --git a/results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json b/results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json
new file mode 100644
index 0000000..6efdfc0
--- /dev/null
+++ b/results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json
@@ -0,0 +1,161 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 984,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.7592328096690931
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.6560231916527999
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.6369652595018086
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.6226169812679291
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.5835219937876651
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.5426512888858193
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.5217562883778623
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.48723151345002025
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.47428832192169995
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.44532516134412664
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.42935120507290486
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.42167684423296076
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.40679138453383196
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.4035281177570945
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.38851159240070143
+ }
+ ],
+ "epochs_completed": 15,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 56.102,
+ "mean_return": 0.125265625,
+ "success": 0.142
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.6426662347128858,
+ "loss": 0.9326626555276822
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 44869632,
+ "cuda_visible_devices": "7",
+ "device": "cuda:0",
+ "git_commit": "17a4224239458781155acb91cfb5f9ea71844663",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 33.518,
+ "mean_return": 0.480853125,
+ "success": 0.522
+ },
+ "stage": "babyai_shared_b0",
+ "training": {
+ "batch_size": 256,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 675335,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 132.41971135139465,
+ "training_wall_seconds": 119.43818664550781
+ },
+ "validation": {
+ "accuracy": 0.8384222888150968,
+ "loss": 0.417832959197432
+ }
+}
diff --git a/results/babyai_shared/pickup_p0_analysis.json b/results/babyai_shared/pickup_p0_analysis.json
new file mode 100644
index 0000000..5c21b79
--- /dev/null
+++ b/results/babyai_shared/pickup_p0_analysis.json
@@ -0,0 +1,115 @@
+{
+ "best_clean_candidate_for_diagnosis": {
+ "bp_action_accuracy_percent": 84.81412488863691,
+ "bp_mission_lesion_drop_points": 52.0,
+ "bp_mission_lesion_success_percent": 11.600000000000001,
+ "bp_rollout_success_percent": 63.6,
+ "checks": {
+ "both_finite": true,
+ "bp_mission_lesion_drop_at_least_20": true,
+ "bp_success_at_least_70": false,
+ "clean_kp_success_at_least_60": false
+ },
+ "clean_kp_action_accuracy_percent": 83.84222888150968,
+ "clean_kp_rollout_success_percent": 52.2,
+ "eligible": false,
+ "hidden_layers": 4,
+ "learning_rate": 0.03,
+ "source_files": [
+ "results/babyai_shared/pickup_p0/d4_lr0.03_bp.json",
+ "results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json"
+ ]
+ },
+ "candidates": [
+ {
+ "bp_action_accuracy_percent": 81.9875273345752,
+ "bp_mission_lesion_drop_points": 34.199999999999996,
+ "bp_mission_lesion_success_percent": 15.6,
+ "bp_rollout_success_percent": 49.8,
+ "checks": {
+ "both_finite": true,
+ "bp_mission_lesion_drop_at_least_20": true,
+ "bp_success_at_least_70": false,
+ "clean_kp_success_at_least_60": false
+ },
+ "clean_kp_action_accuracy_percent": 80.30290758888799,
+ "clean_kp_rollout_success_percent": 45.6,
+ "eligible": false,
+ "hidden_layers": 2,
+ "learning_rate": 0.01,
+ "source_files": [
+ "results/babyai_shared/pickup_p0/d2_lr0.01_bp.json",
+ "results/babyai_shared/pickup_p0/d2_lr0.01_clean_kp.json"
+ ]
+ },
+ {
+ "bp_action_accuracy_percent": 79.03134364622986,
+ "bp_mission_lesion_drop_points": 31.4,
+ "bp_mission_lesion_success_percent": 13.0,
+ "bp_rollout_success_percent": 44.4,
+ "checks": {
+ "both_finite": true,
+ "bp_mission_lesion_drop_at_least_20": true,
+ "bp_success_at_least_70": false,
+ "clean_kp_success_at_least_60": false
+ },
+ "clean_kp_action_accuracy_percent": 80.88604519316434,
+ "clean_kp_rollout_success_percent": 47.0,
+ "eligible": false,
+ "hidden_layers": 2,
+ "learning_rate": 0.03,
+ "source_files": [
+ "results/babyai_shared/pickup_p0/d2_lr0.03_bp.json",
+ "results/babyai_shared/pickup_p0/d2_lr0.03_clean_kp.json"
+ ]
+ },
+ {
+ "bp_action_accuracy_percent": 84.88701708917145,
+ "bp_mission_lesion_drop_points": 51.199999999999996,
+ "bp_mission_lesion_success_percent": 11.600000000000001,
+ "bp_rollout_success_percent": 62.8,
+ "checks": {
+ "both_finite": true,
+ "bp_mission_lesion_drop_at_least_20": true,
+ "bp_success_at_least_70": false,
+ "clean_kp_success_at_least_60": false
+ },
+ "clean_kp_action_accuracy_percent": 81.9308334008261,
+ "clean_kp_rollout_success_percent": 51.0,
+ "eligible": false,
+ "hidden_layers": 4,
+ "learning_rate": 0.01,
+ "source_files": [
+ "results/babyai_shared/pickup_p0/d4_lr0.01_bp.json",
+ "results/babyai_shared/pickup_p0/d4_lr0.01_clean_kp.json"
+ ]
+ },
+ {
+ "bp_action_accuracy_percent": 84.81412488863691,
+ "bp_mission_lesion_drop_points": 52.0,
+ "bp_mission_lesion_success_percent": 11.600000000000001,
+ "bp_rollout_success_percent": 63.6,
+ "checks": {
+ "both_finite": true,
+ "bp_mission_lesion_drop_at_least_20": true,
+ "bp_success_at_least_70": false,
+ "clean_kp_success_at_least_60": false
+ },
+ "clean_kp_action_accuracy_percent": 83.84222888150968,
+ "clean_kp_rollout_success_percent": 52.2,
+ "eligible": false,
+ "hidden_layers": 4,
+ "learning_rate": 0.03,
+ "source_files": [
+ "results/babyai_shared/pickup_p0/d4_lr0.03_bp.json",
+ "results/babyai_shared/pickup_p0/d4_lr0.03_clean_kp.json"
+ ]
+ }
+ ],
+ "decision": "Add task memory and repeat a clean-only gate; do not run raw or SDIL on the present feedforward policy.",
+ "gate": "fail",
+ "raw_or_sdil_results_run_or_read": false,
+ "selected": null,
+ "stage": "babyai_pickup_p0_clean_feasibility",
+ "test_split_generated_or_read": false
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-15 10:14:12 +0000