diff options
| -rw-r--r-- | BABYAI_SHARED_FEEDBACK.md | 14 | ||||
| -rw-r--r-- | experiments/analyze_babyai_shared_b0.py | 86 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d2_lr0.01_bp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d2_lr0.01_clean_kp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d2_lr0.03_bp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d2_lr0.03_clean_kp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d4_lr0.01_bp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d4_lr0.01_clean_kp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d4_lr0.03_bp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0/d4_lr0.03_clean_kp.json | 151 | ||||
| -rw-r--r-- | results/babyai_shared/b0_selector.json | 105 |
11 files changed, 1412 insertions, 1 deletions
diff --git a/BABYAI_SHARED_FEEDBACK.md b/BABYAI_SHARED_FEEDBACK.md index addd58a..d00af72 100644 --- a/BABYAI_SHARED_FEEDBACK.md +++ b/BABYAI_SHARED_FEEDBACK.md @@ -88,7 +88,8 @@ candidate is eligible, B0 fails and the shared-path endpoint is not run. ## B1 mechanism endpoint The selected depth and learning rate are frozen. B1 trains for 40 epochs on -three model seeds with identical data and minibatch orders across conditions: +model and minibatch-order seeds `4101`, `4102`, and `4103`, with identical data +and minibatch orders across conditions: 1. `bp`: exact backpropagation reference with the same fixed mission pathway. 2. `clean_kp`: instruction-only reciprocal KP; this receives a separate clean @@ -114,3 +115,14 @@ Passing B1 opens `PickupLoc` and `PutNextLocalS6N4` with the same method and selection rule, followed by depth scaling. Failure is retained as evidence that task-required shared feedback alone is insufficient to make residualization useful in this setting. + +## B0 outcome + +All eight clean-selector runs were finite. BP and clean KP both reached +`97.8%` validation rollout success and `99.74%` expert-action accuracy in every +candidate. The two-layer, `0.01` learning-rate candidate was selected by the +frozen tie rule: its BP mission lesion reduced rollout success from `97.8%` to +`70.4%`, while it tied for the best clean-KP success and action accuracy and +used the fewest layers and smaller learning rate. The selector read no raw, +SDIL, or test result. This outcome opens B1 without establishing an SDIL +advantage. diff --git a/experiments/analyze_babyai_shared_b0.py b/experiments/analyze_babyai_shared_b0.py new file mode 100644 index 0000000..98cfae8 --- /dev/null +++ b/experiments/analyze_babyai_shared_b0.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Apply the frozen BabyAI B0 clean-selector rule.""" + +import argparse +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_RESULTS = ROOT / "results" / "babyai_shared" / "b0" +DEFAULT_OUT = ROOT / "results" / "babyai_shared" / "b0_selector.json" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--results", type=Path, default=DEFAULT_RESULTS) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = parser.parse_args() + rows = [] + for depth in (2, 4): + for learning_rate in (0.01, 0.03): + records = {} + for condition in ("bp", "clean_kp"): + path = args.results / ( + f"d{depth}_lr{learning_rate}_{condition}.json") + with open(path, encoding="utf-8") as handle: + records[condition] = json.load(handle) + bp = records["bp"] + kp = records["clean_kp"] + bp_success = float(bp["rollout"]["success"]) + kp_success = float(kp["rollout"]["success"]) + bp_lesion = float(bp["mission_lesion_rollout"]["success"]) + checks = { + "both_finite": bool(bp["finite"] and kp["finite"]), + "bp_success_at_least_0p8": bp_success >= 0.8, + "clean_kp_success_at_least_0p8": kp_success >= 0.8, + "bp_mission_lesion_drop_at_least_0p2": ( + bp_success - bp_lesion >= 0.2), + } + rows.append({ + "hidden_layers": depth, + "learning_rate": learning_rate, + "bp_rollout_success": bp_success, + "clean_kp_rollout_success": kp_success, + "bp_mission_lesion_success": bp_lesion, + "bp_mission_lesion_drop": bp_success - bp_lesion, + "bp_action_accuracy": float(bp["validation"]["accuracy"]), + "clean_kp_action_accuracy": float( + kp["validation"]["accuracy"]), + "eligible": all(checks.values()), + "checks": checks, + "source_files": [str( + (args.results / f"d{depth}_lr{learning_rate}_{condition}.json") + .relative_to(ROOT)) for condition in ("bp", "clean_kp")], + }) + eligible = [row for row in rows if row["eligible"]] + selected = max(eligible, key=lambda row: ( + row["clean_kp_rollout_success"], row["clean_kp_action_accuracy"], + -row["hidden_layers"], -row["learning_rate"])) if eligible else None + report = { + "stage": "babyai_shared_b0_selector", + "gate": "pass" if selected is not None else "fail", + "selection_rule": ( + "highest clean-KP rollout success, then action accuracy, then " + "fewer layers, then smaller learning rate, among eligible rows"), + "candidates": rows, + "selected": ({ + "hidden_layers": selected["hidden_layers"], + "width": 256, + "learning_rate": selected["learning_rate"], + "context_gain": 1.0, + "b1_epochs": 40, + "b1_model_and_shuffle_seeds": [4101, 4102, 4103], + } if selected is not None else None), + "raw_or_sdil_results_read": False, + "test_split_generated_or_read": False, + } + args.out.parent.mkdir(parents=True, exist_ok=True) + with open(args.out, "w", encoding="utf-8") as handle: + json.dump(report, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/results/babyai_shared/b0/d2_lr0.01_bp.json b/results/babyai_shared/b0/d2_lr0.01_bp.json new file mode 100644 index 0000000..1e60df3 --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.01_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.35939537301659585 + }, + { + "epoch": 2, + "train_loss": 0.04168066847730767 + }, + { + "epoch": 3, + "train_loss": 0.01859026498753916 + }, + { + "epoch": 4, + "train_loss": 0.015395146521993659 + }, + { + "epoch": 5, + "train_loss": 0.015227786766534503 + }, + { + "epoch": 6, + "train_loss": 0.012962078500369734 + }, + { + "epoch": 7, + "train_loss": 0.013404185661241751 + }, + { + "epoch": 8, + "train_loss": 0.01239592205690728 + }, + { + "epoch": 9, + "train_loss": 0.012198219955157996 + }, + { + "epoch": 10, + "train_loss": 0.011407826012000442 + }, + { + "epoch": 11, + "train_loss": 0.010740509365939281 + }, + { + "epoch": 12, + "train_loss": 0.01174285890576853 + }, + { + "epoch": 13, + "train_loss": 0.012136201211251318 + }, + { + "epoch": 14, + "train_loss": 0.010752951244569637 + }, + { + "epoch": 15, + "train_loss": 0.010369447311809794 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 13.1, + "mean_return": 0.6429, + "success": 0.704 + }, + "mission_lesion_validation": { + "accuracy": 0.8139336087761789, + "loss": 0.5671851808553681 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "0", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 72.15492010116577, + "training_wall_seconds": 68.66148805618286 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.010374719048008347 + } +} diff --git a/results/babyai_shared/b0/d2_lr0.01_clean_kp.json b/results/babyai_shared/b0/d2_lr0.01_clean_kp.json new file mode 100644 index 0000000..c63545b --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.01_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.42220023420724 + }, + { + "epoch": 2, + "train_loss": 0.042795064787973056 + }, + { + "epoch": 3, + "train_loss": 0.01790234833363105 + }, + { + "epoch": 4, + "train_loss": 0.014970682340420106 + }, + { + "epoch": 5, + "train_loss": 0.014731389162638648 + }, + { + "epoch": 6, + "train_loss": 0.012619410693137482 + }, + { + "epoch": 7, + "train_loss": 0.013174676087642596 + }, + { + "epoch": 8, + "train_loss": 0.012047174987383186 + }, + { + "epoch": 9, + "train_loss": 0.011912513411900198 + }, + { + "epoch": 10, + "train_loss": 0.011198682525110516 + }, + { + "epoch": 11, + "train_loss": 0.010507440414618362 + }, + { + "epoch": 12, + "train_loss": 0.011464962014843795 + }, + { + "epoch": 13, + "train_loss": 0.011738500667905266 + }, + { + "epoch": 14, + "train_loss": 0.010477929167999802 + }, + { + "epoch": 15, + "train_loss": 0.010263334540861913 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 16.362, + "mean_return": 0.5511499999999999, + "success": 0.602 + }, + "mission_lesion_validation": { + "accuracy": 0.7602222538823195, + "loss": 0.7746106897906575 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "1", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 72.49147534370422, + "training_wall_seconds": 68.89583253860474 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.010123796248337468 + } +} diff --git a/results/babyai_shared/b0/d2_lr0.03_bp.json b/results/babyai_shared/b0/d2_lr0.03_bp.json new file mode 100644 index 0000000..53c0cd2 --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.03_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.32622500834817236 + }, + { + "epoch": 2, + "train_loss": 0.022967058995907955 + }, + { + "epoch": 3, + "train_loss": 0.013182067105292596 + }, + { + "epoch": 4, + "train_loss": 0.12631320300893012 + }, + { + "epoch": 5, + "train_loss": 0.03265200278869915 + }, + { + "epoch": 6, + "train_loss": 0.01237200420434502 + }, + { + "epoch": 7, + "train_loss": 0.012444034099012655 + }, + { + "epoch": 8, + "train_loss": 0.010830031347418712 + }, + { + "epoch": 9, + "train_loss": 0.010618501521848057 + }, + { + "epoch": 10, + "train_loss": 0.010414972044315867 + }, + { + "epoch": 11, + "train_loss": 0.009751175390768118 + }, + { + "epoch": 12, + "train_loss": 0.01019353245279159 + }, + { + "epoch": 13, + "train_loss": 0.010311937543394213 + }, + { + "epoch": 14, + "train_loss": 0.00966560988549397 + }, + { + "epoch": 15, + "train_loss": 0.009808915172311986 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 6.608, + "mean_return": 0.8254000000000001, + "success": 0.906 + }, + "mission_lesion_validation": { + "accuracy": 0.9393075936743126, + "loss": 0.15966886549462006 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "2", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 71.35738158226013, + "training_wall_seconds": 68.60933923721313 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.008090585890548625 + } +} diff --git a/results/babyai_shared/b0/d2_lr0.03_clean_kp.json b/results/babyai_shared/b0/d2_lr0.03_clean_kp.json new file mode 100644 index 0000000..767f65d --- /dev/null +++ b/results/babyai_shared/b0/d2_lr0.03_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 2, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.4284296942739324 + }, + { + "epoch": 2, + "train_loss": 0.021993979699909686 + }, + { + "epoch": 3, + "train_loss": 0.012347337700511244 + }, + { + "epoch": 4, + "train_loss": 0.11120144955475222 + }, + { + "epoch": 5, + "train_loss": 0.02621290213089775 + }, + { + "epoch": 6, + "train_loss": 0.011413470651869748 + }, + { + "epoch": 7, + "train_loss": 0.011871577144404661 + }, + { + "epoch": 8, + "train_loss": 0.010163592805302787 + }, + { + "epoch": 9, + "train_loss": 0.009837346972728317 + }, + { + "epoch": 10, + "train_loss": 0.01011049080259082 + }, + { + "epoch": 11, + "train_loss": 0.009574851958419789 + }, + { + "epoch": 12, + "train_loss": 0.009629912125899202 + }, + { + "epoch": 13, + "train_loss": 0.009786839098352092 + }, + { + "epoch": 14, + "train_loss": 0.009409503974602558 + }, + { + "epoch": 15, + "train_loss": 0.009612279576884413 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 5.856, + "mean_return": 0.8466, + "success": 0.93 + }, + "mission_lesion_validation": { + "accuracy": 0.9532696965379683, + "loss": 0.12229915629222635 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 39800832, + "cuda_visible_devices": "3", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 394759, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 71.55634665489197, + "training_wall_seconds": 68.94226169586182 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.007343835429191453 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.01_bp.json b/results/babyai_shared/b0/d4_lr0.01_bp.json new file mode 100644 index 0000000..ce30242 --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.01_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.3253778777474707 + }, + { + "epoch": 2, + "train_loss": 0.019206965685060078 + }, + { + "epoch": 3, + "train_loss": 0.012563892805271528 + }, + { + "epoch": 4, + "train_loss": 0.013036079497330568 + }, + { + "epoch": 5, + "train_loss": 0.014585806006692688 + }, + { + "epoch": 6, + "train_loss": 0.011469631964256141 + }, + { + "epoch": 7, + "train_loss": 0.012590554792894346 + }, + { + "epoch": 8, + "train_loss": 0.010635647804857316 + }, + { + "epoch": 9, + "train_loss": 0.01044608271402963 + }, + { + "epoch": 10, + "train_loss": 0.01032080053330653 + }, + { + "epoch": 11, + "train_loss": 0.009672622701830485 + }, + { + "epoch": 12, + "train_loss": 0.010132916131353175 + }, + { + "epoch": 13, + "train_loss": 0.010232909059956332 + }, + { + "epoch": 14, + "train_loss": 0.009590742462103001 + }, + { + "epoch": 15, + "train_loss": 0.009766413487642157 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 18.046, + "mean_return": 0.50385, + "success": 0.55 + }, + "mission_lesion_validation": { + "accuracy": 0.7350049864653084, + "loss": 1.0839614618162532 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "4", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 69.9526629447937, + "training_wall_seconds": 63.36152243614197 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.007651511259713494 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.01_clean_kp.json b/results/babyai_shared/b0/d4_lr0.01_clean_kp.json new file mode 100644 index 0000000..72a132c --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.01_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.01, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.01, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.4667839081585407 + }, + { + "epoch": 2, + "train_loss": 0.04674517553976991 + }, + { + "epoch": 3, + "train_loss": 0.01326043784279715 + }, + { + "epoch": 4, + "train_loss": 0.061285746785896746 + }, + { + "epoch": 5, + "train_loss": 0.026032158348878676 + }, + { + "epoch": 6, + "train_loss": 0.013545795751904899 + }, + { + "epoch": 7, + "train_loss": 0.014207788907065564 + }, + { + "epoch": 8, + "train_loss": 0.011456064975143156 + }, + { + "epoch": 9, + "train_loss": 0.011107088818402685 + }, + { + "epoch": 10, + "train_loss": 0.010615441516486251 + }, + { + "epoch": 11, + "train_loss": 0.009885513414840468 + }, + { + "epoch": 12, + "train_loss": 0.01040836228650402 + }, + { + "epoch": 13, + "train_loss": 0.010366036330180411 + }, + { + "epoch": 14, + "train_loss": 0.009639627484350719 + }, + { + "epoch": 15, + "train_loss": 0.009855725453976033 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 4.292, + "mean_return": 0.8905000000000001, + "success": 0.978 + }, + "mission_lesion_validation": { + "accuracy": 0.9955834164410885, + "loss": 0.01790462883301042 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "5", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 68.48475790023804, + "training_wall_seconds": 64.21018767356873 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.007562428802009797 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.03_bp.json b/results/babyai_shared/b0/d4_lr0.03_bp.json new file mode 100644 index 0000000..33f8418 --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.03_bp.json @@ -0,0 +1,151 @@ +{ + "condition": "bp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.5505747761509635 + }, + { + "epoch": 2, + "train_loss": 0.3316267966343598 + }, + { + "epoch": 3, + "train_loss": 0.09943052434563553 + }, + { + "epoch": 4, + "train_loss": 0.04950741482962092 + }, + { + "epoch": 5, + "train_loss": 0.013397140696906718 + }, + { + "epoch": 6, + "train_loss": 0.010083013286560097 + }, + { + "epoch": 7, + "train_loss": 0.011028704402918017 + }, + { + "epoch": 8, + "train_loss": 0.009480555155737834 + }, + { + "epoch": 9, + "train_loss": 0.009246192916672788 + }, + { + "epoch": 10, + "train_loss": 0.009409724382448158 + }, + { + "epoch": 11, + "train_loss": 0.00913212353257801 + }, + { + "epoch": 12, + "train_loss": 0.00898012409410016 + }, + { + "epoch": 13, + "train_loss": 0.009049554331206971 + }, + { + "epoch": 14, + "train_loss": 0.00897477681505155 + }, + { + "epoch": 15, + "train_loss": 0.009097340858795426 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "mission_lesion_validation": { + "accuracy": 0.9974355321270836, + "loss": 0.009770111168767503 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "6", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 67.55784368515015, + "training_wall_seconds": 63.23557496070862 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.00700099701928284 + } +} diff --git a/results/babyai_shared/b0/d4_lr0.03_clean_kp.json b/results/babyai_shared/b0/d4_lr0.03_clean_kp.json new file mode 100644 index 0000000..b0c853d --- /dev/null +++ b/results/babyai_shared/b0/d4_lr0.03_clean_kp.json @@ -0,0 +1,151 @@ +{ + "condition": "clean_kp", + "config": { + "action_dim": 7, + "context_gain": 1.0, + "hidden_layers": 4, + "input_dim": 984, + "learning_rate": 0.03, + "mission_dim": 13, + "momentum": 0.9, + "reciprocal_learning_rate": 0.03, + "weight_decay": 0.0001, + "width": 256 + }, + "data": { + "color_cardinality": 6, + "elapsed_seconds": 37.730633020401, + "env_id": "BabyAI-GoToObjS6-v1", + "minigrid_version": "3.1.0", + "object_cardinality": 11, + "protocol": "babyai_shared_feedback_b0", + "rollout_episodes": 500, + "rollout_seed_start": 200000, + "state_cardinality": 3, + "train_episodes": 20000, + "train_steps": 70159, + "validation_episodes": 2000, + "validation_steps": 7019, + "vocabulary": [ + "<unk>", + "ball", + "blue", + "box", + "go", + "green", + "grey", + "key", + "purple", + "red", + "the", + "to", + "yellow" + ] + }, + "epoch_history": [ + { + "epoch": 1, + "train_loss": 0.560454016964544 + }, + { + "epoch": 2, + "train_loss": 0.11413068584420465 + }, + { + "epoch": 3, + "train_loss": 0.01235340728620279 + }, + { + "epoch": 4, + "train_loss": 0.010933047032872723 + }, + { + "epoch": 5, + "train_loss": 0.010575457136702342 + }, + { + "epoch": 6, + "train_loss": 0.00985285716623449 + }, + { + "epoch": 7, + "train_loss": 0.010482580381583168 + }, + { + "epoch": 8, + "train_loss": 0.009581763261918572 + }, + { + "epoch": 9, + "train_loss": 0.009271365735324005 + }, + { + "epoch": 10, + "train_loss": 0.00958018275731857 + }, + { + "epoch": 11, + "train_loss": 0.009324251855247316 + }, + { + "epoch": 12, + "train_loss": 0.00918881041791544 + }, + { + "epoch": 13, + "train_loss": 0.009378983245802705 + }, + { + "epoch": 14, + "train_loss": 0.009324983275675384 + }, + { + "epoch": 15, + "train_loss": 0.009338655431241601 + } + ], + "epochs_completed": 15, + "finite": true, + "first_nonfinite_epoch": null, + "mission_lesion_rollout": { + "episodes": 500, + "mean_length": 4.306, + "mean_return": 0.8901500000000001, + "success": 0.978 + }, + "mission_lesion_validation": { + "accuracy": 0.9921641259438666, + "loss": 0.029844395743083507 + }, + "predictor": [], + "provenance": { + "cuda_device_name": "NVIDIA GeForce GTX 1080", + "cuda_peak_allocated_bytes": 42582016, + "cuda_visible_devices": "7", + "device": "cuda:0", + "git_commit": "20fd3577693593e3f62bcc4e1c2565dca5349944", + "git_dirty_tracked": false, + "minigrid_version": "3.1.0", + "torch_version": "2.3.1+cu118" + }, + "rollout": { + "episodes": 500, + "mean_length": 4.29, + "mean_return": 0.8905500000000001, + "success": 0.978 + }, + "stage": "babyai_shared_b0", + "training": { + "batch_size": 256, + "model_seed": 4101, + "neutral_examples_per_epoch": 0, + "parameter_count_including_reciprocal_context_predictor": 665095, + "shuffle_seed": 4101, + "total_wall_seconds_including_rollouts": 72.05698609352112, + "training_wall_seconds": 69.7202775478363 + }, + "validation": { + "accuracy": 0.9974355321270836, + "loss": 0.006933236702977295 + } +} diff --git a/results/babyai_shared/b0_selector.json b/results/babyai_shared/b0_selector.json new file mode 100644 index 0000000..4368e44 --- /dev/null +++ b/results/babyai_shared/b0_selector.json @@ -0,0 +1,105 @@ +{ + "candidates": [ + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.274, + "bp_mission_lesion_success": 0.704, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": true, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": true, + "hidden_layers": 2, + "learning_rate": 0.01, + "source_files": [ + "results/babyai_shared/b0/d2_lr0.01_bp.json", + "results/babyai_shared/b0/d2_lr0.01_clean_kp.json" + ] + }, + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.07199999999999995, + "bp_mission_lesion_success": 0.906, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": false, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": false, + "hidden_layers": 2, + "learning_rate": 0.03, + "source_files": [ + "results/babyai_shared/b0/d2_lr0.03_bp.json", + "results/babyai_shared/b0/d2_lr0.03_clean_kp.json" + ] + }, + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.42799999999999994, + "bp_mission_lesion_success": 0.55, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": true, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": true, + "hidden_layers": 4, + "learning_rate": 0.01, + "source_files": [ + "results/babyai_shared/b0/d4_lr0.01_bp.json", + "results/babyai_shared/b0/d4_lr0.01_clean_kp.json" + ] + }, + { + "bp_action_accuracy": 0.9974355321270836, + "bp_mission_lesion_drop": 0.0, + "bp_mission_lesion_success": 0.978, + "bp_rollout_success": 0.978, + "checks": { + "both_finite": true, + "bp_mission_lesion_drop_at_least_0p2": false, + "bp_success_at_least_0p8": true, + "clean_kp_success_at_least_0p8": true + }, + "clean_kp_action_accuracy": 0.9974355321270836, + "clean_kp_rollout_success": 0.978, + "eligible": false, + "hidden_layers": 4, + "learning_rate": 0.03, + "source_files": [ + "results/babyai_shared/b0/d4_lr0.03_bp.json", + "results/babyai_shared/b0/d4_lr0.03_clean_kp.json" + ] + } + ], + "gate": "pass", + "raw_or_sdil_results_read": false, + "selected": { + "b1_epochs": 40, + "b1_model_and_shuffle_seeds": [ + 4101, + 4102, + 4103 + ], + "context_gain": 1.0, + "hidden_layers": 2, + "learning_rate": 0.01, + "width": 256 + }, + "selection_rule": "highest clean-KP rollout success, then action accuracy, then fewer layers, then smaller learning rate, among eligible rows", + "stage": "babyai_shared_b0_selector", + "test_split_generated_or_read": false +} |