summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--BABYAI_SHARED_FEEDBACK.md10
-rw-r--r--experiments/analyze_babyai_pickup_p1.py77
-rw-r--r--results/babyai_shared/pickup_p1/h4_bp.json202
-rw-r--r--results/babyai_shared/pickup_p1/h4_clean_kp.json202
-rw-r--r--results/babyai_shared/pickup_p1/h8_bp.json202
-rw-r--r--results/babyai_shared/pickup_p1/h8_clean_kp.json202
-rw-r--r--results/babyai_shared/pickup_p1_selector.json53
7 files changed, 948 insertions, 0 deletions
diff --git a/BABYAI_SHARED_FEEDBACK.md b/BABYAI_SHARED_FEEDBACK.md
index 9443253..009709f 100644
--- a/BABYAI_SHARED_FEEDBACK.md
+++ b/BABYAI_SHARED_FEEDBACK.md
@@ -184,3 +184,13 @@ history lengths four and eight for 25 epochs, using only BP and clean KP. The
same `70%` BP, `60%` clean-KP, and 20-point BP mission-lesion thresholds apply;
ties favor the shorter history. This changes the missing state information,
not the shared-feedback signal or its strength.
+
+## P1 outcome
+
+Four-step history passed: BP reached `77.6%` rollout success, clean KP reached
+`61.6%`, and removing the mission reduced BP success to `44.6%`. Eight-step
+history did not pass (`67.2%` BP and `42.2%` clean KP), so the frozen selector
+chooses four steps. The P2 mechanism endpoint uses four hidden layers, width
+256, learning rate `0.03`, four-step history, 40 epochs, and model/minibatch
+seeds `4101`, `4102`, and `4103`. No raw or SDIL result was run or read during
+selection.
diff --git a/experiments/analyze_babyai_pickup_p1.py b/experiments/analyze_babyai_pickup_p1.py
new file mode 100644
index 0000000..5a63896
--- /dev/null
+++ b/experiments/analyze_babyai_pickup_p1.py
@@ -0,0 +1,77 @@
+#!/usr/bin/env python3
+"""Apply the frozen PickupLoc history selector."""
+
+import argparse
+import json
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_RESULTS = ROOT / "results" / "babyai_shared" / "pickup_p1"
+DEFAULT_OUT = ROOT / "results" / "babyai_shared" / "pickup_p1_selector.json"
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS)
+ parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+ args = parser.parse_args()
+ candidates = []
+ for history_steps in (4, 8):
+ records = {}
+ for condition in ("bp", "clean_kp"):
+ path = args.results / f"h{history_steps}_{condition}.json"
+ with open(path, encoding="utf-8") as handle:
+ records[condition] = json.load(handle)
+ bp, kp = records["bp"], records["clean_kp"]
+ bp_success = 100.0 * bp["rollout"]["success"]
+ kp_success = 100.0 * kp["rollout"]["success"]
+ lesion = 100.0 * bp["mission_lesion_rollout"]["success"]
+ checks = {
+ "both_finite": bool(bp["finite"] and kp["finite"]),
+ "bp_success_at_least_70": bp_success >= 70.0,
+ "clean_kp_success_at_least_60": kp_success >= 60.0,
+ "bp_mission_lesion_drop_at_least_20": bp_success - lesion >= 20.0,
+ }
+ candidates.append({
+ "history_steps": history_steps,
+ "bp_rollout_success_percent": bp_success,
+ "clean_kp_rollout_success_percent": kp_success,
+ "bp_mission_lesion_success_percent": lesion,
+ "bp_mission_lesion_drop_points": bp_success - lesion,
+ "bp_action_accuracy_percent": 100.0 * bp["validation"]["accuracy"],
+ "clean_kp_action_accuracy_percent": (
+ 100.0 * kp["validation"]["accuracy"]),
+ "eligible": all(checks.values()),
+ "checks": checks,
+ })
+ eligible = [row for row in candidates if row["eligible"]]
+ selected = max(eligible, key=lambda row: (
+ row["clean_kp_rollout_success_percent"],
+ row["clean_kp_action_accuracy_percent"],
+ -row["history_steps"])) if eligible else None
+ report = {
+ "stage": "babyai_pickup_p1_history_selector",
+ "gate": "pass" if selected is not None else "fail",
+ "candidates": candidates,
+ "selected": ({
+ "history_steps": selected["history_steps"],
+ "hidden_layers": 4,
+ "width": 256,
+ "learning_rate": 0.03,
+ "context_gain": 1.0,
+ "p2_epochs": 40,
+ "p2_model_and_shuffle_seeds": [4101, 4102, 4103],
+ } if selected is not None else None),
+ "raw_or_sdil_results_run_or_read": False,
+ "test_split_generated_or_read": False,
+ }
+ args.out.parent.mkdir(parents=True, exist_ok=True)
+ with open(args.out, "w", encoding="utf-8") as handle:
+ json.dump(report, handle, indent=2, sort_keys=True)
+ handle.write("\n")
+ print(json.dumps(report, indent=2, sort_keys=True))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/results/babyai_shared/pickup_p1/h4_bp.json b/results/babyai_shared/pickup_p1/h4_bp.json
new file mode 100644
index 0000000..7f50692
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h4_bp.json
@@ -0,0 +1,202 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 3964,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.617732492434351
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.4960194272116611
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.46592118231873764
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.4488015041225835
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.43406150341033933
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.40718337021375955
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.3807649670776568
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.3537148001319484
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.32643107909905283
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.3063848673356207
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.28984395764375986
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.2761740297392795
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.2647564570841036
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.25636729594908264
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.24771365335113124
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.2439696610287616
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.2316527129474439
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.22913515367006002
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.21849248581811
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.21554886654803626
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.20946172889910247
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.20310457860168657
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.2004946485946053
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.19238460020015113
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.19454976933567147
+ }
+ ],
+ "epochs_completed": 25,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 44.738,
+ "mean_return": 0.315471875,
+ "success": 0.446
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.7258443346561918,
+ "loss": 0.7968140768408147
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 130979840,
+ "cuda_visible_devices": "0",
+ "device": "cuda:0",
+ "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 24.48,
+ "mean_return": 0.6333499999999999,
+ "success": 0.776
+ },
+ "stage": "babyai_pickup_p1",
+ "training": {
+ "batch_size": 256,
+ "history_steps": 4,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 1438215,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 107.65545964241028,
+ "training_wall_seconds": 93.02058959007263
+ },
+ "validation": {
+ "accuracy": 0.8726006317324047,
+ "loss": 0.29367609611839346
+ }
+}
diff --git a/results/babyai_shared/pickup_p1/h4_clean_kp.json b/results/babyai_shared/pickup_p1/h4_clean_kp.json
new file mode 100644
index 0000000..5b5c237
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h4_clean_kp.json
@@ -0,0 +1,202 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 3964,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.6793887242517973
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.5070055065029546
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.4780606180743167
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.46464681211270786
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.45651326932405173
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.4414772361203244
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.4295881901916705
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.4249380529554267
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.4178563620542225
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.40975865175849513
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.40546370631770084
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.3982252176811821
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.3858343799490678
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.379007538933503
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.3656208556576779
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.35379995308424295
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.33681938353337737
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.32208908896697197
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.30617407221543164
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.2926546273419732
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.28012341188757045
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.27170656586948194
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.2618562983211718
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.25525567559819473
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.24340780433855558
+ }
+ ],
+ "epochs_completed": 25,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 44.094,
+ "mean_return": 0.32912812499999994,
+ "success": 0.492
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.7498177694986636,
+ "loss": 0.6379798919437983
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 130979840,
+ "cuda_visible_devices": "1",
+ "device": "cuda:0",
+ "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 31.056,
+ "mean_return": 0.524875,
+ "success": 0.616
+ },
+ "stage": "babyai_pickup_p1",
+ "training": {
+ "batch_size": 256,
+ "history_steps": 4,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 1438215,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 112.63866543769836,
+ "training_wall_seconds": 98.61020755767822
+ },
+ "validation": {
+ "accuracy": 0.8578602089576415,
+ "loss": 0.31587319402276837
+ }
+}
diff --git a/results/babyai_shared/pickup_p1/h8_bp.json b/results/babyai_shared/pickup_p1/h8_bp.json
new file mode 100644
index 0000000..bc9f7e5
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h8_bp.json
@@ -0,0 +1,202 @@
+{
+ "condition": "bp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 7928,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.6321194964961002
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.4924550595408992
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.4738408164601577
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.44879562810847634
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.4285990457785757
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.41075110887226307
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.39140965926019766
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.37369739294052123
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.34568895095273067
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.3196536517456958
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.30388873991213344
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.28137334092667227
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.27270261155931574
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.2571609906773818
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.24464017152786255
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.2372008582792784
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.23206563582545833
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.22314524976830732
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.21708365371352747
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.20607339666077965
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.20372903077225937
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.19777495506562684
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.19298629737214038
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.18727699292333502
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.18384965098217915
+ }
+ ],
+ "epochs_completed": 25,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 46.88,
+ "mean_return": 0.27814999999999995,
+ "success": 0.374
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.7470640641451365,
+ "loss": 0.7762952407400666
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 247702528,
+ "cuda_visible_devices": "2",
+ "device": "cuda:0",
+ "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 28.428,
+ "mean_return": 0.56743125,
+ "success": 0.672
+ },
+ "stage": "babyai_pickup_p1",
+ "training": {
+ "batch_size": 256,
+ "history_steps": 8,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 2452999,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 113.33854031562805,
+ "training_wall_seconds": 97.784170627594
+ },
+ "validation": {
+ "accuracy": 0.8668502470235685,
+ "loss": 0.32782987784767786
+ }
+}
diff --git a/results/babyai_shared/pickup_p1/h8_clean_kp.json b/results/babyai_shared/pickup_p1/h8_clean_kp.json
new file mode 100644
index 0000000..f51ec11
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h8_clean_kp.json
@@ -0,0 +1,202 @@
+{
+ "condition": "clean_kp",
+ "config": {
+ "action_dim": 7,
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "input_dim": 7928,
+ "learning_rate": 0.03,
+ "mission_dim": 23,
+ "momentum": 0.9,
+ "reciprocal_learning_rate": 0.03,
+ "weight_decay": 0.0001,
+ "width": 256
+ },
+ "data": {
+ "color_cardinality": 6,
+ "elapsed_seconds": 81.82925486564636,
+ "env_id": "BabyAI-PickupLoc-v0",
+ "minigrid_version": "3.1.0",
+ "object_cardinality": 11,
+ "protocol": "babyai_shared_feedback_b0",
+ "rollout_episodes": 500,
+ "rollout_seed_start": 200000,
+ "state_cardinality": 3,
+ "train_episodes": 20000,
+ "train_steps": 121467,
+ "validation_episodes": 2000,
+ "validation_steps": 12347,
+ "vocabulary": [
+ "<unk>",
+ "a",
+ "ball",
+ "behind",
+ "blue",
+ "box",
+ "front",
+ "green",
+ "grey",
+ "in",
+ "key",
+ "left",
+ "of",
+ "on",
+ "pick",
+ "purple",
+ "red",
+ "right",
+ "the",
+ "up",
+ "yellow",
+ "you",
+ "your"
+ ]
+ },
+ "epoch_history": [
+ {
+ "epoch": 1,
+ "train_loss": 0.6952196103648135
+ },
+ {
+ "epoch": 2,
+ "train_loss": 0.5047668682274066
+ },
+ {
+ "epoch": 3,
+ "train_loss": 0.4749976762344963
+ },
+ {
+ "epoch": 4,
+ "train_loss": 0.4631779912898415
+ },
+ {
+ "epoch": 5,
+ "train_loss": 0.45011724729286995
+ },
+ {
+ "epoch": 6,
+ "train_loss": 0.43483660158358123
+ },
+ {
+ "epoch": 7,
+ "train_loss": 0.4248962173963848
+ },
+ {
+ "epoch": 8,
+ "train_loss": 0.4174242968935716
+ },
+ {
+ "epoch": 9,
+ "train_loss": 0.41010714556041516
+ },
+ {
+ "epoch": 10,
+ "train_loss": 0.4014566550756756
+ },
+ {
+ "epoch": 11,
+ "train_loss": 0.39507218191498206
+ },
+ {
+ "epoch": 12,
+ "train_loss": 0.3851292588836268
+ },
+ {
+ "epoch": 13,
+ "train_loss": 0.3778882456453223
+ },
+ {
+ "epoch": 14,
+ "train_loss": 0.3718921935558319
+ },
+ {
+ "epoch": 15,
+ "train_loss": 0.36200399285868595
+ },
+ {
+ "epoch": 16,
+ "train_loss": 0.3529288729868437
+ },
+ {
+ "epoch": 17,
+ "train_loss": 0.34455470483554035
+ },
+ {
+ "epoch": 18,
+ "train_loss": 0.3405523705168774
+ },
+ {
+ "epoch": 19,
+ "train_loss": 0.33339980376394174
+ },
+ {
+ "epoch": 20,
+ "train_loss": 0.32586026505420085
+ },
+ {
+ "epoch": 21,
+ "train_loss": 0.32184638095529455
+ },
+ {
+ "epoch": 22,
+ "train_loss": 0.3121714940510298
+ },
+ {
+ "epoch": 23,
+ "train_loss": 0.30236716722187246
+ },
+ {
+ "epoch": 24,
+ "train_loss": 0.2920462249768408
+ },
+ {
+ "epoch": 25,
+ "train_loss": 0.28397564828395844
+ }
+ ],
+ "epochs_completed": 25,
+ "finite": true,
+ "first_nonfinite_epoch": null,
+ "mission_lesion_rollout": {
+ "episodes": 500,
+ "mean_length": 48.154,
+ "mean_return": 0.25483437499999995,
+ "success": 0.32
+ },
+ "mission_lesion_validation": {
+ "accuracy": 0.7601846602413542,
+ "loss": 0.5676966533010533
+ },
+ "predictor": [],
+ "provenance": {
+ "cuda_device_name": "NVIDIA GeForce GTX 1080",
+ "cuda_peak_allocated_bytes": 247702528,
+ "cuda_visible_devices": "3",
+ "device": "cuda:0",
+ "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+ "git_dirty_tracked": false,
+ "minigrid_version": "3.1.0",
+ "torch_version": "2.3.1+cu118"
+ },
+ "rollout": {
+ "episodes": 500,
+ "mean_length": 40.478,
+ "mean_return": 0.372978125,
+ "success": 0.422
+ },
+ "stage": "babyai_pickup_p1",
+ "training": {
+ "batch_size": 256,
+ "history_steps": 8,
+ "model_seed": 4101,
+ "neutral_examples_per_epoch": 0,
+ "parameter_count_including_reciprocal_context_predictor": 2452999,
+ "shuffle_seed": 4101,
+ "total_wall_seconds_including_rollouts": 117.14025735855103,
+ "training_wall_seconds": 101.26001691818237
+ },
+ "validation": {
+ "accuracy": 0.8334818174455333,
+ "loss": 0.37354338703014944
+ }
+}
diff --git a/results/babyai_shared/pickup_p1_selector.json b/results/babyai_shared/pickup_p1_selector.json
new file mode 100644
index 0000000..2f5fc07
--- /dev/null
+++ b/results/babyai_shared/pickup_p1_selector.json
@@ -0,0 +1,53 @@
+{
+ "candidates": [
+ {
+ "bp_action_accuracy_percent": 87.26006317324047,
+ "bp_mission_lesion_drop_points": 33.00000000000001,
+ "bp_mission_lesion_success_percent": 44.6,
+ "bp_rollout_success_percent": 77.60000000000001,
+ "checks": {
+ "both_finite": true,
+ "bp_mission_lesion_drop_at_least_20": true,
+ "bp_success_at_least_70": true,
+ "clean_kp_success_at_least_60": true
+ },
+ "clean_kp_action_accuracy_percent": 85.78602089576415,
+ "clean_kp_rollout_success_percent": 61.6,
+ "eligible": true,
+ "history_steps": 4
+ },
+ {
+ "bp_action_accuracy_percent": 86.68502470235686,
+ "bp_mission_lesion_drop_points": 29.800000000000004,
+ "bp_mission_lesion_success_percent": 37.4,
+ "bp_rollout_success_percent": 67.2,
+ "checks": {
+ "both_finite": true,
+ "bp_mission_lesion_drop_at_least_20": true,
+ "bp_success_at_least_70": false,
+ "clean_kp_success_at_least_60": false
+ },
+ "clean_kp_action_accuracy_percent": 83.34818174455333,
+ "clean_kp_rollout_success_percent": 42.199999999999996,
+ "eligible": false,
+ "history_steps": 8
+ }
+ ],
+ "gate": "pass",
+ "raw_or_sdil_results_run_or_read": false,
+ "selected": {
+ "context_gain": 1.0,
+ "hidden_layers": 4,
+ "history_steps": 4,
+ "learning_rate": 0.03,
+ "p2_epochs": 40,
+ "p2_model_and_shuffle_seeds": [
+ 4101,
+ 4102,
+ 4103
+ ],
+ "width": 256
+ },
+ "stage": "babyai_pickup_p1_history_selector",
+ "test_split_generated_or_read": false
+}
generated by cgit v1.2.3 (git 2.25.1) at 2026-09-15 11:45:40 +0000