results: select four-step PickupLoc history

5 files changed, 861 insertions, 0 deletions

diff --git a/results/babyai_shared/pickup_p1/h4_bp.json b/results/babyai_shared/pickup_p1/h4_bp.json
new file mode 100644
index 0000000..7f50692
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h4_bp.json
@@ -0,0 +1,202 @@
+{
+  "condition": "bp",
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 3964,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "data": {
+    "color_cardinality": 6,
+    "elapsed_seconds": 81.82925486564636,
+    "env_id": "BabyAI-PickupLoc-v0",
+    "minigrid_version": "3.1.0",
+    "object_cardinality": 11,
+    "protocol": "babyai_shared_feedback_b0",
+    "rollout_episodes": 500,
+    "rollout_seed_start": 200000,
+    "state_cardinality": 3,
+    "train_episodes": 20000,
+    "train_steps": 121467,
+    "validation_episodes": 2000,
+    "validation_steps": 12347,
+    "vocabulary": [
+      "<unk>",
+      "a",
+      "ball",
+      "behind",
+      "blue",
+      "box",
+      "front",
+      "green",
+      "grey",
+      "in",
+      "key",
+      "left",
+      "of",
+      "on",
+      "pick",
+      "purple",
+      "red",
+      "right",
+      "the",
+      "up",
+      "yellow",
+      "you",
+      "your"
+    ]
+  },
+  "epoch_history": [
+    {
+      "epoch": 1,
+      "train_loss": 0.617732492434351
+    },
+    {
+      "epoch": 2,
+      "train_loss": 0.4960194272116611
+    },
+    {
+      "epoch": 3,
+      "train_loss": 0.46592118231873764
+    },
+    {
+      "epoch": 4,
+      "train_loss": 0.4488015041225835
+    },
+    {
+      "epoch": 5,
+      "train_loss": 0.43406150341033933
+    },
+    {
+      "epoch": 6,
+      "train_loss": 0.40718337021375955
+    },
+    {
+      "epoch": 7,
+      "train_loss": 0.3807649670776568
+    },
+    {
+      "epoch": 8,
+      "train_loss": 0.3537148001319484
+    },
+    {
+      "epoch": 9,
+      "train_loss": 0.32643107909905283
+    },
+    {
+      "epoch": 10,
+      "train_loss": 0.3063848673356207
+    },
+    {
+      "epoch": 11,
+      "train_loss": 0.28984395764375986
+    },
+    {
+      "epoch": 12,
+      "train_loss": 0.2761740297392795
+    },
+    {
+      "epoch": 13,
+      "train_loss": 0.2647564570841036
+    },
+    {
+      "epoch": 14,
+      "train_loss": 0.25636729594908264
+    },
+    {
+      "epoch": 15,
+      "train_loss": 0.24771365335113124
+    },
+    {
+      "epoch": 16,
+      "train_loss": 0.2439696610287616
+    },
+    {
+      "epoch": 17,
+      "train_loss": 0.2316527129474439
+    },
+    {
+      "epoch": 18,
+      "train_loss": 0.22913515367006002
+    },
+    {
+      "epoch": 19,
+      "train_loss": 0.21849248581811
+    },
+    {
+      "epoch": 20,
+      "train_loss": 0.21554886654803626
+    },
+    {
+      "epoch": 21,
+      "train_loss": 0.20946172889910247
+    },
+    {
+      "epoch": 22,
+      "train_loss": 0.20310457860168657
+    },
+    {
+      "epoch": 23,
+      "train_loss": 0.2004946485946053
+    },
+    {
+      "epoch": 24,
+      "train_loss": 0.19238460020015113
+    },
+    {
+      "epoch": 25,
+      "train_loss": 0.19454976933567147
+    }
+  ],
+  "epochs_completed": 25,
+  "finite": true,
+  "first_nonfinite_epoch": null,
+  "mission_lesion_rollout": {
+    "episodes": 500,
+    "mean_length": 44.738,
+    "mean_return": 0.315471875,
+    "success": 0.446
+  },
+  "mission_lesion_validation": {
+    "accuracy": 0.7258443346561918,
+    "loss": 0.7968140768408147
+  },
+  "predictor": [],
+  "provenance": {
+    "cuda_device_name": "NVIDIA GeForce GTX 1080",
+    "cuda_peak_allocated_bytes": 130979840,
+    "cuda_visible_devices": "0",
+    "device": "cuda:0",
+    "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+    "git_dirty_tracked": false,
+    "minigrid_version": "3.1.0",
+    "torch_version": "2.3.1+cu118"
+  },
+  "rollout": {
+    "episodes": 500,
+    "mean_length": 24.48,
+    "mean_return": 0.6333499999999999,
+    "success": 0.776
+  },
+  "stage": "babyai_pickup_p1",
+  "training": {
+    "batch_size": 256,
+    "history_steps": 4,
+    "model_seed": 4101,
+    "neutral_examples_per_epoch": 0,
+    "parameter_count_including_reciprocal_context_predictor": 1438215,
+    "shuffle_seed": 4101,
+    "total_wall_seconds_including_rollouts": 107.65545964241028,
+    "training_wall_seconds": 93.02058959007263
+  },
+  "validation": {
+    "accuracy": 0.8726006317324047,
+    "loss": 0.29367609611839346
+  }
+}
diff --git a/results/babyai_shared/pickup_p1/h4_clean_kp.json b/results/babyai_shared/pickup_p1/h4_clean_kp.json
new file mode 100644
index 0000000..5b5c237
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h4_clean_kp.json
@@ -0,0 +1,202 @@
+{
+  "condition": "clean_kp",
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 3964,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "data": {
+    "color_cardinality": 6,
+    "elapsed_seconds": 81.82925486564636,
+    "env_id": "BabyAI-PickupLoc-v0",
+    "minigrid_version": "3.1.0",
+    "object_cardinality": 11,
+    "protocol": "babyai_shared_feedback_b0",
+    "rollout_episodes": 500,
+    "rollout_seed_start": 200000,
+    "state_cardinality": 3,
+    "train_episodes": 20000,
+    "train_steps": 121467,
+    "validation_episodes": 2000,
+    "validation_steps": 12347,
+    "vocabulary": [
+      "<unk>",
+      "a",
+      "ball",
+      "behind",
+      "blue",
+      "box",
+      "front",
+      "green",
+      "grey",
+      "in",
+      "key",
+      "left",
+      "of",
+      "on",
+      "pick",
+      "purple",
+      "red",
+      "right",
+      "the",
+      "up",
+      "yellow",
+      "you",
+      "your"
+    ]
+  },
+  "epoch_history": [
+    {
+      "epoch": 1,
+      "train_loss": 0.6793887242517973
+    },
+    {
+      "epoch": 2,
+      "train_loss": 0.5070055065029546
+    },
+    {
+      "epoch": 3,
+      "train_loss": 0.4780606180743167
+    },
+    {
+      "epoch": 4,
+      "train_loss": 0.46464681211270786
+    },
+    {
+      "epoch": 5,
+      "train_loss": 0.45651326932405173
+    },
+    {
+      "epoch": 6,
+      "train_loss": 0.4414772361203244
+    },
+    {
+      "epoch": 7,
+      "train_loss": 0.4295881901916705
+    },
+    {
+      "epoch": 8,
+      "train_loss": 0.4249380529554267
+    },
+    {
+      "epoch": 9,
+      "train_loss": 0.4178563620542225
+    },
+    {
+      "epoch": 10,
+      "train_loss": 0.40975865175849513
+    },
+    {
+      "epoch": 11,
+      "train_loss": 0.40546370631770084
+    },
+    {
+      "epoch": 12,
+      "train_loss": 0.3982252176811821
+    },
+    {
+      "epoch": 13,
+      "train_loss": 0.3858343799490678
+    },
+    {
+      "epoch": 14,
+      "train_loss": 0.379007538933503
+    },
+    {
+      "epoch": 15,
+      "train_loss": 0.3656208556576779
+    },
+    {
+      "epoch": 16,
+      "train_loss": 0.35379995308424295
+    },
+    {
+      "epoch": 17,
+      "train_loss": 0.33681938353337737
+    },
+    {
+      "epoch": 18,
+      "train_loss": 0.32208908896697197
+    },
+    {
+      "epoch": 19,
+      "train_loss": 0.30617407221543164
+    },
+    {
+      "epoch": 20,
+      "train_loss": 0.2926546273419732
+    },
+    {
+      "epoch": 21,
+      "train_loss": 0.28012341188757045
+    },
+    {
+      "epoch": 22,
+      "train_loss": 0.27170656586948194
+    },
+    {
+      "epoch": 23,
+      "train_loss": 0.2618562983211718
+    },
+    {
+      "epoch": 24,
+      "train_loss": 0.25525567559819473
+    },
+    {
+      "epoch": 25,
+      "train_loss": 0.24340780433855558
+    }
+  ],
+  "epochs_completed": 25,
+  "finite": true,
+  "first_nonfinite_epoch": null,
+  "mission_lesion_rollout": {
+    "episodes": 500,
+    "mean_length": 44.094,
+    "mean_return": 0.32912812499999994,
+    "success": 0.492
+  },
+  "mission_lesion_validation": {
+    "accuracy": 0.7498177694986636,
+    "loss": 0.6379798919437983
+  },
+  "predictor": [],
+  "provenance": {
+    "cuda_device_name": "NVIDIA GeForce GTX 1080",
+    "cuda_peak_allocated_bytes": 130979840,
+    "cuda_visible_devices": "1",
+    "device": "cuda:0",
+    "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+    "git_dirty_tracked": false,
+    "minigrid_version": "3.1.0",
+    "torch_version": "2.3.1+cu118"
+  },
+  "rollout": {
+    "episodes": 500,
+    "mean_length": 31.056,
+    "mean_return": 0.524875,
+    "success": 0.616
+  },
+  "stage": "babyai_pickup_p1",
+  "training": {
+    "batch_size": 256,
+    "history_steps": 4,
+    "model_seed": 4101,
+    "neutral_examples_per_epoch": 0,
+    "parameter_count_including_reciprocal_context_predictor": 1438215,
+    "shuffle_seed": 4101,
+    "total_wall_seconds_including_rollouts": 112.63866543769836,
+    "training_wall_seconds": 98.61020755767822
+  },
+  "validation": {
+    "accuracy": 0.8578602089576415,
+    "loss": 0.31587319402276837
+  }
+}
diff --git a/results/babyai_shared/pickup_p1/h8_bp.json b/results/babyai_shared/pickup_p1/h8_bp.json
new file mode 100644
index 0000000..bc9f7e5
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h8_bp.json
@@ -0,0 +1,202 @@
+{
+  "condition": "bp",
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 7928,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "data": {
+    "color_cardinality": 6,
+    "elapsed_seconds": 81.82925486564636,
+    "env_id": "BabyAI-PickupLoc-v0",
+    "minigrid_version": "3.1.0",
+    "object_cardinality": 11,
+    "protocol": "babyai_shared_feedback_b0",
+    "rollout_episodes": 500,
+    "rollout_seed_start": 200000,
+    "state_cardinality": 3,
+    "train_episodes": 20000,
+    "train_steps": 121467,
+    "validation_episodes": 2000,
+    "validation_steps": 12347,
+    "vocabulary": [
+      "<unk>",
+      "a",
+      "ball",
+      "behind",
+      "blue",
+      "box",
+      "front",
+      "green",
+      "grey",
+      "in",
+      "key",
+      "left",
+      "of",
+      "on",
+      "pick",
+      "purple",
+      "red",
+      "right",
+      "the",
+      "up",
+      "yellow",
+      "you",
+      "your"
+    ]
+  },
+  "epoch_history": [
+    {
+      "epoch": 1,
+      "train_loss": 0.6321194964961002
+    },
+    {
+      "epoch": 2,
+      "train_loss": 0.4924550595408992
+    },
+    {
+      "epoch": 3,
+      "train_loss": 0.4738408164601577
+    },
+    {
+      "epoch": 4,
+      "train_loss": 0.44879562810847634
+    },
+    {
+      "epoch": 5,
+      "train_loss": 0.4285990457785757
+    },
+    {
+      "epoch": 6,
+      "train_loss": 0.41075110887226307
+    },
+    {
+      "epoch": 7,
+      "train_loss": 0.39140965926019766
+    },
+    {
+      "epoch": 8,
+      "train_loss": 0.37369739294052123
+    },
+    {
+      "epoch": 9,
+      "train_loss": 0.34568895095273067
+    },
+    {
+      "epoch": 10,
+      "train_loss": 0.3196536517456958
+    },
+    {
+      "epoch": 11,
+      "train_loss": 0.30388873991213344
+    },
+    {
+      "epoch": 12,
+      "train_loss": 0.28137334092667227
+    },
+    {
+      "epoch": 13,
+      "train_loss": 0.27270261155931574
+    },
+    {
+      "epoch": 14,
+      "train_loss": 0.2571609906773818
+    },
+    {
+      "epoch": 15,
+      "train_loss": 0.24464017152786255
+    },
+    {
+      "epoch": 16,
+      "train_loss": 0.2372008582792784
+    },
+    {
+      "epoch": 17,
+      "train_loss": 0.23206563582545833
+    },
+    {
+      "epoch": 18,
+      "train_loss": 0.22314524976830732
+    },
+    {
+      "epoch": 19,
+      "train_loss": 0.21708365371352747
+    },
+    {
+      "epoch": 20,
+      "train_loss": 0.20607339666077965
+    },
+    {
+      "epoch": 21,
+      "train_loss": 0.20372903077225937
+    },
+    {
+      "epoch": 22,
+      "train_loss": 0.19777495506562684
+    },
+    {
+      "epoch": 23,
+      "train_loss": 0.19298629737214038
+    },
+    {
+      "epoch": 24,
+      "train_loss": 0.18727699292333502
+    },
+    {
+      "epoch": 25,
+      "train_loss": 0.18384965098217915
+    }
+  ],
+  "epochs_completed": 25,
+  "finite": true,
+  "first_nonfinite_epoch": null,
+  "mission_lesion_rollout": {
+    "episodes": 500,
+    "mean_length": 46.88,
+    "mean_return": 0.27814999999999995,
+    "success": 0.374
+  },
+  "mission_lesion_validation": {
+    "accuracy": 0.7470640641451365,
+    "loss": 0.7762952407400666
+  },
+  "predictor": [],
+  "provenance": {
+    "cuda_device_name": "NVIDIA GeForce GTX 1080",
+    "cuda_peak_allocated_bytes": 247702528,
+    "cuda_visible_devices": "2",
+    "device": "cuda:0",
+    "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+    "git_dirty_tracked": false,
+    "minigrid_version": "3.1.0",
+    "torch_version": "2.3.1+cu118"
+  },
+  "rollout": {
+    "episodes": 500,
+    "mean_length": 28.428,
+    "mean_return": 0.56743125,
+    "success": 0.672
+  },
+  "stage": "babyai_pickup_p1",
+  "training": {
+    "batch_size": 256,
+    "history_steps": 8,
+    "model_seed": 4101,
+    "neutral_examples_per_epoch": 0,
+    "parameter_count_including_reciprocal_context_predictor": 2452999,
+    "shuffle_seed": 4101,
+    "total_wall_seconds_including_rollouts": 113.33854031562805,
+    "training_wall_seconds": 97.784170627594
+  },
+  "validation": {
+    "accuracy": 0.8668502470235685,
+    "loss": 0.32782987784767786
+  }
+}
diff --git a/results/babyai_shared/pickup_p1/h8_clean_kp.json b/results/babyai_shared/pickup_p1/h8_clean_kp.json
new file mode 100644
index 0000000..f51ec11
--- /dev/null
+++ b/results/babyai_shared/pickup_p1/h8_clean_kp.json
@@ -0,0 +1,202 @@
+{
+  "condition": "clean_kp",
+  "config": {
+    "action_dim": 7,
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "input_dim": 7928,
+    "learning_rate": 0.03,
+    "mission_dim": 23,
+    "momentum": 0.9,
+    "reciprocal_learning_rate": 0.03,
+    "weight_decay": 0.0001,
+    "width": 256
+  },
+  "data": {
+    "color_cardinality": 6,
+    "elapsed_seconds": 81.82925486564636,
+    "env_id": "BabyAI-PickupLoc-v0",
+    "minigrid_version": "3.1.0",
+    "object_cardinality": 11,
+    "protocol": "babyai_shared_feedback_b0",
+    "rollout_episodes": 500,
+    "rollout_seed_start": 200000,
+    "state_cardinality": 3,
+    "train_episodes": 20000,
+    "train_steps": 121467,
+    "validation_episodes": 2000,
+    "validation_steps": 12347,
+    "vocabulary": [
+      "<unk>",
+      "a",
+      "ball",
+      "behind",
+      "blue",
+      "box",
+      "front",
+      "green",
+      "grey",
+      "in",
+      "key",
+      "left",
+      "of",
+      "on",
+      "pick",
+      "purple",
+      "red",
+      "right",
+      "the",
+      "up",
+      "yellow",
+      "you",
+      "your"
+    ]
+  },
+  "epoch_history": [
+    {
+      "epoch": 1,
+      "train_loss": 0.6952196103648135
+    },
+    {
+      "epoch": 2,
+      "train_loss": 0.5047668682274066
+    },
+    {
+      "epoch": 3,
+      "train_loss": 0.4749976762344963
+    },
+    {
+      "epoch": 4,
+      "train_loss": 0.4631779912898415
+    },
+    {
+      "epoch": 5,
+      "train_loss": 0.45011724729286995
+    },
+    {
+      "epoch": 6,
+      "train_loss": 0.43483660158358123
+    },
+    {
+      "epoch": 7,
+      "train_loss": 0.4248962173963848
+    },
+    {
+      "epoch": 8,
+      "train_loss": 0.4174242968935716
+    },
+    {
+      "epoch": 9,
+      "train_loss": 0.41010714556041516
+    },
+    {
+      "epoch": 10,
+      "train_loss": 0.4014566550756756
+    },
+    {
+      "epoch": 11,
+      "train_loss": 0.39507218191498206
+    },
+    {
+      "epoch": 12,
+      "train_loss": 0.3851292588836268
+    },
+    {
+      "epoch": 13,
+      "train_loss": 0.3778882456453223
+    },
+    {
+      "epoch": 14,
+      "train_loss": 0.3718921935558319
+    },
+    {
+      "epoch": 15,
+      "train_loss": 0.36200399285868595
+    },
+    {
+      "epoch": 16,
+      "train_loss": 0.3529288729868437
+    },
+    {
+      "epoch": 17,
+      "train_loss": 0.34455470483554035
+    },
+    {
+      "epoch": 18,
+      "train_loss": 0.3405523705168774
+    },
+    {
+      "epoch": 19,
+      "train_loss": 0.33339980376394174
+    },
+    {
+      "epoch": 20,
+      "train_loss": 0.32586026505420085
+    },
+    {
+      "epoch": 21,
+      "train_loss": 0.32184638095529455
+    },
+    {
+      "epoch": 22,
+      "train_loss": 0.3121714940510298
+    },
+    {
+      "epoch": 23,
+      "train_loss": 0.30236716722187246
+    },
+    {
+      "epoch": 24,
+      "train_loss": 0.2920462249768408
+    },
+    {
+      "epoch": 25,
+      "train_loss": 0.28397564828395844
+    }
+  ],
+  "epochs_completed": 25,
+  "finite": true,
+  "first_nonfinite_epoch": null,
+  "mission_lesion_rollout": {
+    "episodes": 500,
+    "mean_length": 48.154,
+    "mean_return": 0.25483437499999995,
+    "success": 0.32
+  },
+  "mission_lesion_validation": {
+    "accuracy": 0.7601846602413542,
+    "loss": 0.5676966533010533
+  },
+  "predictor": [],
+  "provenance": {
+    "cuda_device_name": "NVIDIA GeForce GTX 1080",
+    "cuda_peak_allocated_bytes": 247702528,
+    "cuda_visible_devices": "3",
+    "device": "cuda:0",
+    "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100",
+    "git_dirty_tracked": false,
+    "minigrid_version": "3.1.0",
+    "torch_version": "2.3.1+cu118"
+  },
+  "rollout": {
+    "episodes": 500,
+    "mean_length": 40.478,
+    "mean_return": 0.372978125,
+    "success": 0.422
+  },
+  "stage": "babyai_pickup_p1",
+  "training": {
+    "batch_size": 256,
+    "history_steps": 8,
+    "model_seed": 4101,
+    "neutral_examples_per_epoch": 0,
+    "parameter_count_including_reciprocal_context_predictor": 2452999,
+    "shuffle_seed": 4101,
+    "total_wall_seconds_including_rollouts": 117.14025735855103,
+    "training_wall_seconds": 101.26001691818237
+  },
+  "validation": {
+    "accuracy": 0.8334818174455333,
+    "loss": 0.37354338703014944
+  }
+}
diff --git a/results/babyai_shared/pickup_p1_selector.json b/results/babyai_shared/pickup_p1_selector.json
new file mode 100644
index 0000000..2f5fc07
--- /dev/null
+++ b/results/babyai_shared/pickup_p1_selector.json
@@ -0,0 +1,53 @@
+{
+  "candidates": [
+    {
+      "bp_action_accuracy_percent": 87.26006317324047,
+      "bp_mission_lesion_drop_points": 33.00000000000001,
+      "bp_mission_lesion_success_percent": 44.6,
+      "bp_rollout_success_percent": 77.60000000000001,
+      "checks": {
+        "both_finite": true,
+        "bp_mission_lesion_drop_at_least_20": true,
+        "bp_success_at_least_70": true,
+        "clean_kp_success_at_least_60": true
+      },
+      "clean_kp_action_accuracy_percent": 85.78602089576415,
+      "clean_kp_rollout_success_percent": 61.6,
+      "eligible": true,
+      "history_steps": 4
+    },
+    {
+      "bp_action_accuracy_percent": 86.68502470235686,
+      "bp_mission_lesion_drop_points": 29.800000000000004,
+      "bp_mission_lesion_success_percent": 37.4,
+      "bp_rollout_success_percent": 67.2,
+      "checks": {
+        "both_finite": true,
+        "bp_mission_lesion_drop_at_least_20": true,
+        "bp_success_at_least_70": false,
+        "clean_kp_success_at_least_60": false
+      },
+      "clean_kp_action_accuracy_percent": 83.34818174455333,
+      "clean_kp_rollout_success_percent": 42.199999999999996,
+      "eligible": false,
+      "history_steps": 8
+    }
+  ],
+  "gate": "pass",
+  "raw_or_sdil_results_run_or_read": false,
+  "selected": {
+    "context_gain": 1.0,
+    "hidden_layers": 4,
+    "history_steps": 4,
+    "learning_rate": 0.03,
+    "p2_epochs": 40,
+    "p2_model_and_shuffle_seeds": [
+      4101,
+      4102,
+      4103
+    ],
+    "width": 256
+  },
+  "stage": "babyai_pickup_p1_history_selector",
+  "test_split_generated_or_read": false
+}