From 7b7d7a08a5e78c3379397f01f0c9af97d5d84fbd Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Mon, 10 Aug 2026 10:34:53 -0500 Subject: results: select four-step PickupLoc history --- results/babyai_shared/pickup_p1/h4_bp.json | 202 +++++++++++++++++++++++ results/babyai_shared/pickup_p1/h4_clean_kp.json | 202 +++++++++++++++++++++++ results/babyai_shared/pickup_p1/h8_bp.json | 202 +++++++++++++++++++++++ results/babyai_shared/pickup_p1/h8_clean_kp.json | 202 +++++++++++++++++++++++ results/babyai_shared/pickup_p1_selector.json | 53 ++++++ 5 files changed, 861 insertions(+) create mode 100644 results/babyai_shared/pickup_p1/h4_bp.json create mode 100644 results/babyai_shared/pickup_p1/h4_clean_kp.json create mode 100644 results/babyai_shared/pickup_p1/h8_bp.json create mode 100644 results/babyai_shared/pickup_p1/h8_clean_kp.json create mode 100644 results/babyai_shared/pickup_p1_selector.json (limited to 'results') diff --git a/results/babyai_shared/pickup_p1/h4_bp.json b/results/babyai_shared/pickup_p1/h4_bp.json new file mode 100644 index 0000000..7f50692 --- /dev/null +++ b/results/babyai_shared/pickup_p1/h4_bp.json @@ -0,0 +1,202 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 3964, + "learning_rate": 0.03, + "mission_dim": 23, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 81.82925486564636, + "env_id": "BabyAI-PickupLoc-v0", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 121467, + "validation_episodes": 2000, + "validation_steps": 12347, + "vocabulary": [ + "", + "a", + "ball", + "behind", + "blue", + "box", + "front", + "green", + "grey", + "in", + "key", + "left", + "of", + "on", + "pick", + "purple", + "red", + "right", + "the", + "up", + "yellow", + "you", + "your" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.617732492434351 + }, + { + "epoch": 2, + "train_loss": 0.4960194272116611 + }, + { + "epoch": 3, + "train_loss": 0.46592118231873764 + }, + { + "epoch": 4, + "train_loss": 0.4488015041225835 + }, + { + "epoch": 5, + "train_loss": 0.43406150341033933 + }, + { + "epoch": 6, + "train_loss": 0.40718337021375955 + }, + { + "epoch": 7, + "train_loss": 0.3807649670776568 + }, + { + "epoch": 8, + "train_loss": 0.3537148001319484 + }, + { + "epoch": 9, + "train_loss": 0.32643107909905283 + }, + { + "epoch": 10, + "train_loss": 0.3063848673356207 + }, + { + "epoch": 11, + "train_loss": 0.28984395764375986 + }, + { + "epoch": 12, + "train_loss": 0.2761740297392795 + }, + { + "epoch": 13, + "train_loss": 0.2647564570841036 + }, + { + "epoch": 14, + "train_loss": 0.25636729594908264 + }, + { + "epoch": 15, + "train_loss": 0.24771365335113124 + }, + { + "epoch": 16, + "train_loss": 0.2439696610287616 + }, + { + "epoch": 17, + "train_loss": 0.2316527129474439 + }, + { + "epoch": 18, + "train_loss": 0.22913515367006002 + }, + { + "epoch": 19, + "train_loss": 0.21849248581811 + }, + { + "epoch": 20, + "train_loss": 0.21554886654803626 + }, + { + "epoch": 21, + "train_loss": 0.20946172889910247 + }, + { + "epoch": 22, + "train_loss": 0.20310457860168657 + }, + { + "epoch": 23, + "train_loss": 0.2004946485946053 + }, + { + "epoch": 24, + "train_loss": 0.19238460020015113 + }, + { + "epoch": 25, + "train_loss": 0.19454976933567147 + } + ], + "epochs_completed": 25, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 44.738, + "mean_return": 0.315471875, + "success": 0.446 + }, + "mission_lesion_validation": { + "accuracy": 0.7258443346561918, + "loss": 0.7968140768408147 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 130979840, + "cuda_visible_devices": "0", + "device": "cuda:0", + "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 24.48, + "mean_return": 0.6333499999999999, + "success": 0.776 + }, + "stage": "babyai_pickup_p1", + "training": { + "batch_size": 256, + "history_steps": 4, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 1438215, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 107.65545964241028, + "training_wall_seconds": 93.02058959007263 + }, + "validation": { + "accuracy": 0.8726006317324047, + "loss": 0.29367609611839346 + } +} diff --git a/results/babyai_shared/pickup_p1/h4_clean_kp.json b/results/babyai_shared/pickup_p1/h4_clean_kp.json new file mode 100644 index 0000000..5b5c237 --- /dev/null +++ b/results/babyai_shared/pickup_p1/h4_clean_kp.json @@ -0,0 +1,202 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 3964, + "learning_rate": 0.03, + "mission_dim": 23, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 81.82925486564636, + "env_id": "BabyAI-PickupLoc-v0", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 121467, + "validation_episodes": 2000, + "validation_steps": 12347, + "vocabulary": [ + "", + "a", + "ball", + "behind", + "blue", + "box", + "front", + "green", + "grey", + "in", + "key", + "left", + "of", + "on", + "pick", + "purple", + "red", + "right", + "the", + "up", + "yellow", + "you", + "your" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.6793887242517973 + }, + { + "epoch": 2, + "train_loss": 0.5070055065029546 + }, + { + "epoch": 3, + "train_loss": 0.4780606180743167 + }, + { + "epoch": 4, + "train_loss": 0.46464681211270786 + }, + { + "epoch": 5, + "train_loss": 0.45651326932405173 + }, + { + "epoch": 6, + "train_loss": 0.4414772361203244 + }, + { + "epoch": 7, + "train_loss": 0.4295881901916705 + }, + { + "epoch": 8, + "train_loss": 0.4249380529554267 + }, + { + "epoch": 9, + "train_loss": 0.4178563620542225 + }, + { + "epoch": 10, + "train_loss": 0.40975865175849513 + }, + { + "epoch": 11, + "train_loss": 0.40546370631770084 + }, + { + "epoch": 12, + "train_loss": 0.3982252176811821 + }, + { + "epoch": 13, + "train_loss": 0.3858343799490678 + }, + { + "epoch": 14, + "train_loss": 0.379007538933503 + }, + { + "epoch": 15, + "train_loss": 0.3656208556576779 + }, + { + "epoch": 16, + "train_loss": 0.35379995308424295 + }, + { + "epoch": 17, + "train_loss": 0.33681938353337737 + }, + { + "epoch": 18, + "train_loss": 0.32208908896697197 + }, + { + "epoch": 19, + "train_loss": 0.30617407221543164 + }, + { + "epoch": 20, + "train_loss": 0.2926546273419732 + }, + { + "epoch": 21, + "train_loss": 0.28012341188757045 + }, + { + "epoch": 22, + "train_loss": 0.27170656586948194 + }, + { + "epoch": 23, + "train_loss": 0.2618562983211718 + }, + { + "epoch": 24, + "train_loss": 0.25525567559819473 + }, + { + "epoch": 25, + "train_loss": 0.24340780433855558 + } + ], + "epochs_completed": 25, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 44.094, + "mean_return": 0.32912812499999994, + "success": 0.492 + }, + "mission_lesion_validation": { + "accuracy": 0.7498177694986636, + "loss": 0.6379798919437983 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 130979840, + "cuda_visible_devices": "1", + "device": "cuda:0", + "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 31.056, + "mean_return": 0.524875, + "success": 0.616 + }, + "stage": "babyai_pickup_p1", + "training": { + "batch_size": 256, + "history_steps": 4, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 1438215, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 112.63866543769836, + "training_wall_seconds": 98.61020755767822 + }, + "validation": { + "accuracy": 0.8578602089576415, + "loss": 0.31587319402276837 + } +} diff --git a/results/babyai_shared/pickup_p1/h8_bp.json b/results/babyai_shared/pickup_p1/h8_bp.json new file mode 100644 index 0000000..bc9f7e5 --- /dev/null +++ b/results/babyai_shared/pickup_p1/h8_bp.json @@ -0,0 +1,202 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 7928, + "learning_rate": 0.03, + "mission_dim": 23, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 81.82925486564636, + "env_id": "BabyAI-PickupLoc-v0", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 121467, + "validation_episodes": 2000, + "validation_steps": 12347, + "vocabulary": [ + "", + "a", + "ball", + "behind", + "blue", + "box", + "front", + "green", + "grey", + "in", + "key", + "left", + "of", + "on", + "pick", + "purple", + "red", + "right", + "the", + "up", + "yellow", + "you", + "your" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.6321194964961002 + }, + { + "epoch": 2, + "train_loss": 0.4924550595408992 + }, + { + "epoch": 3, + "train_loss": 0.4738408164601577 + }, + { + "epoch": 4, + "train_loss": 0.44879562810847634 + }, + { + "epoch": 5, + "train_loss": 0.4285990457785757 + }, + { + "epoch": 6, + "train_loss": 0.41075110887226307 + }, + { + "epoch": 7, + "train_loss": 0.39140965926019766 + }, + { + "epoch": 8, + "train_loss": 0.37369739294052123 + }, + { + "epoch": 9, + "train_loss": 0.34568895095273067 + }, + { + "epoch": 10, + "train_loss": 0.3196536517456958 + }, + { + "epoch": 11, + "train_loss": 0.30388873991213344 + }, + { + "epoch": 12, + "train_loss": 0.28137334092667227 + }, + { + "epoch": 13, + "train_loss": 0.27270261155931574 + }, + { + "epoch": 14, + "train_loss": 0.2571609906773818 + }, + { + "epoch": 15, + "train_loss": 0.24464017152786255 + }, + { + "epoch": 16, + "train_loss": 0.2372008582792784 + }, + { + "epoch": 17, + "train_loss": 0.23206563582545833 + }, + { + "epoch": 18, + "train_loss": 0.22314524976830732 + }, + { + "epoch": 19, + "train_loss": 0.21708365371352747 + }, + { + "epoch": 20, + "train_loss": 0.20607339666077965 + }, + { + "epoch": 21, + "train_loss": 0.20372903077225937 + }, + { + "epoch": 22, + "train_loss": 0.19777495506562684 + }, + { + "epoch": 23, + "train_loss": 0.19298629737214038 + }, + { + "epoch": 24, + "train_loss": 0.18727699292333502 + }, + { + "epoch": 25, + "train_loss": 0.18384965098217915 + } + ], + "epochs_completed": 25, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 46.88, + "mean_return": 0.27814999999999995, + "success": 0.374 + }, + "mission_lesion_validation": { + "accuracy": 0.7470640641451365, + "loss": 0.7762952407400666 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 247702528, + "cuda_visible_devices": "2", + "device": "cuda:0", + "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 28.428, + "mean_return": 0.56743125, + "success": 0.672 + }, + "stage": "babyai_pickup_p1", + "training": { + "batch_size": 256, + "history_steps": 8, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 2452999, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 113.33854031562805, + "training_wall_seconds": 97.784170627594 + }, + "validation": { + "accuracy": 0.8668502470235685, + "loss": 0.32782987784767786 + } +} diff --git a/results/babyai_shared/pickup_p1/h8_clean_kp.json b/results/babyai_shared/pickup_p1/h8_clean_kp.json new file mode 100644 index 0000000..f51ec11 --- /dev/null +++ b/results/babyai_shared/pickup_p1/h8_clean_kp.json @@ -0,0 +1,202 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 7928, + "learning_rate": 0.03, + "mission_dim": 23, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 81.82925486564636, + "env_id": "BabyAI-PickupLoc-v0", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 121467, + "validation_episodes": 2000, + "validation_steps": 12347, + "vocabulary": [ + "", + "a", + "ball", + "behind", + "blue", + "box", + "front", + "green", + "grey", + "in", + "key", + "left", + "of", + "on", + "pick", + "purple", + "red", + "right", + "the", + "up", + "yellow", + "you", + "your" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.6952196103648135 + }, + { + "epoch": 2, + "train_loss": 0.5047668682274066 + }, + { + "epoch": 3, + "train_loss": 0.4749976762344963 + }, + { + "epoch": 4, + "train_loss": 0.4631779912898415 + }, + { + "epoch": 5, + "train_loss": 0.45011724729286995 + }, + { + "epoch": 6, + "train_loss": 0.43483660158358123 + }, + { + "epoch": 7, + "train_loss": 0.4248962173963848 + }, + { + "epoch": 8, + "train_loss": 0.4174242968935716 + }, + { + "epoch": 9, + "train_loss": 0.41010714556041516 + }, + { + "epoch": 10, + "train_loss": 0.4014566550756756 + }, + { + "epoch": 11, + "train_loss": 0.39507218191498206 + }, + { + "epoch": 12, + "train_loss": 0.3851292588836268 + }, + { + "epoch": 13, + "train_loss": 0.3778882456453223 + }, + { + "epoch": 14, + "train_loss": 0.3718921935558319 + }, + { + "epoch": 15, + "train_loss": 0.36200399285868595 + }, + { + "epoch": 16, + "train_loss": 0.3529288729868437 + }, + { + "epoch": 17, + "train_loss": 0.34455470483554035 + }, + { + "epoch": 18, + "train_loss": 0.3405523705168774 + }, + { + "epoch": 19, + "train_loss": 0.33339980376394174 + }, + { + "epoch": 20, + "train_loss": 0.32586026505420085 + }, + { + "epoch": 21, + "train_loss": 0.32184638095529455 + }, + { + "epoch": 22, + "train_loss": 0.3121714940510298 + }, + { + "epoch": 23, + "train_loss": 0.30236716722187246 + }, + { + "epoch": 24, + "train_loss": 0.2920462249768408 + }, + { + "epoch": 25, + "train_loss": 0.28397564828395844 + } + ], + "epochs_completed": 25, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 48.154, + "mean_return": 0.25483437499999995, + "success": 0.32 + }, + "mission_lesion_validation": { + "accuracy": 0.7601846602413542, + "loss": 0.5676966533010533 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 247702528, + "cuda_visible_devices": "3", + "device": "cuda:0", + "git_commit": "a30c297eb8abf6f15f97dd4f35a2f530c9728100", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 40.478, + "mean_return": 0.372978125, + "success": 0.422 + }, + "stage": "babyai_pickup_p1", + "training": { + "batch_size": 256, + "history_steps": 8, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 2452999, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 117.14025735855103, + "training_wall_seconds": 101.26001691818237 + }, + "validation": { + "accuracy": 0.8334818174455333, + "loss": 0.37354338703014944 + } +} diff --git a/results/babyai_shared/pickup_p1_selector.json b/results/babyai_shared/pickup_p1_selector.json new file mode 100644 index 0000000..2f5fc07 --- /dev/null +++ b/results/babyai_shared/pickup_p1_selector.json @@ -0,0 +1,53 @@ +{ + "candidates": [ + { + "bp_action_accuracy_percent": 87.26006317324047, + "bp_mission_lesion_drop_points": 33.00000000000001, + "bp_mission_lesion_success_percent": 44.6, + "bp_rollout_success_percent": 77.60000000000001, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_20": true, + "bp_success_at_least_70": true, + "clean_kp_success_at_least_60": true + }, + "clean_kp_action_accuracy_percent": 85.78602089576415, + "clean_kp_rollout_success_percent": 61.6, + "eligible": true, + "history_steps": 4 + }, + { + "bp_action_accuracy_percent": 86.68502470235686, + "bp_mission_lesion_drop_points": 29.800000000000004, + "bp_mission_lesion_success_percent": 37.4, + "bp_rollout_success_percent": 67.2, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_20": true, + "bp_success_at_least_70": false, + "clean_kp_success_at_least_60": false + }, + "clean_kp_action_accuracy_percent": 83.34818174455333, + "clean_kp_rollout_success_percent": 42.199999999999996, + "eligible": false, + "history_steps": 8 + } + ], + "gate": "pass", + "raw_or_sdil_results_run_or_read": false, + "selected": { + "context_gain": 1.0, + "hidden_layers": 4, + "history_steps": 4, + "learning_rate": 0.03, + "p2_epochs": 40, + "p2_model_and_shuffle_seeds": [ + 4101, + 4102, + 4103 + ], + "width": 256 + }, + "stage": "babyai_pickup_p1_history_selector", + "test_split_generated_or_read": false +} -- cgit v1.2.3