diff options
Diffstat (limited to 'experiments/analyze_bci_v2_development.py')
| -rw-r--r-- | experiments/analyze_bci_v2_development.py | 370 |
1 files changed, 370 insertions, 0 deletions
diff --git a/experiments/analyze_bci_v2_development.py b/experiments/analyze_bci_v2_development.py new file mode 100644 index 0000000..4acf321 --- /dev/null +++ b/experiments/analyze_bci_v2_development.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Audit and select the frozen oral-B-v2 development grid.""" +import argparse +import glob +import hashlib +import json +import os +import statistics + +from experiments.bci_v2_run import ( + CHALLENGE_EPISODES, + CONDITIONS, + CRITIC_ETAS, + FORWARD_ETAS, + GAMMAS, + HORIZONS, + MODEL_SEEDS, + PROTOCOL_PATH, + TASK_SEEDS, + build_config, + source_paths, +) + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RUNNER_PATH = os.path.join(ROOT, "experiments", "bci_v2_run.py") + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def tag(value): + return str(value).replace(".", "p") + + +def filename(candidate, task_seed): + forward_eta, gamma, critic_eta = candidate + return ( + f"bci_v2_e{tag(forward_eta)}_g{tag(gamma)}" + f"_c{tag(critic_eta)}_t{task_seed}_m0.json" + ) + + +def candidate_checks(row): + conditions = row["conditions"] + signatures = row["signatures"] + intact = conditions["intact"] + return { + "final_success_at_least_0p70": + intact["final_success"] >= 0.70, + "learning_gain_at_least_0p10": + intact["learning_gain"] >= 0.10, + "fixed_role_final_gap_at_least_0p20": ( + intact["final_success"] + - conditions["fixed_role"]["final_success"] + ) >= 0.20, + "oracle_final_deficit_at_most_0p10": ( + conditions["oracle_role"]["final_success"] + - intact["final_success"] + ) <= 0.10, + "plasticity_lesion_retains_at_most_half_gain": + conditions["plasticity_lesion"]["learning_gain"] + <= 0.5 * intact["learning_gain"], + "learned_role_cosine_at_least_0p80": + intact["role_cosine_after_training"] >= 0.80, + "challenge_success_fraction_between_0p15_and_0p85": + 0.15 <= signatures["challenge_success_fraction"] <= 0.85, + "terminal_outcome_accuracy_at_least_0p65": + signatures[ + "terminal_residual_outcome_balanced_acc" + ] >= 0.65, + "terminal_role_separation_at_least_0p03": + signatures[ + "terminal_role_aligned_outcome_separation" + ] >= 0.03, + "acute_outcome_lesion_separation_drop_at_least_0p01": + signatures[ + "terminal_outcome_separation_drop_under_acute_lesion" + ] >= 0.01, + "critic_expectedness_contribution_at_least_0p005": + signatures[ + "mean_critic_expectedness_contribution" + ] >= 0.005, + "critic_contribution_tracks_value_at_least_0p95": + signatures[ + "critic_contribution_value_prediction_corr" + ] >= 0.95, + "residual_soma_corr_at_most_0p10": + signatures["mean_abs_residual_soma_corr"] <= 0.10, + "raw_residual_corr_gap_at_least_0p20": + signatures[ + "raw_minus_residual_abs_soma_corr" + ] >= 0.20, + "surrounding_event_accuracy_at_least_0p52": + signatures[ + "surrounding_event_decoder_balanced_acc" + ] >= 0.52, + "decoder_distance_corr_at_least_0p02": + signatures[ + "decoder_distance_residual_corr" + ] >= 0.02, + "sign_inversion_at_least_0p01": + signatures[ + "causal_role_sign_inversion_index" + ] >= 0.01, + "velocity_advantage_at_least_0p05": + signatures[ + "velocity_minus_error_abs_cv_corr" + ] >= 0.05, + } + + +def validate(row, path, candidate, task_seed, digests): + require(row.get("schema_version") == 2, f"{path}: schema") + args = row.get("args", {}) + require(args.get("task_seed") == task_seed, f"{path}: task seed") + require(args.get("model_seed") in MODEL_SEEDS, f"{path}: model seed") + require( + ( + args.get("forward_eta"), + args.get("gamma"), + args.get("critic_eta"), + ) == candidate, + f"{path}: candidate", + ) + protocol = row.get("protocol", {}) + require( + protocol.get("name") == "oral_b_v2_development_v1", + f"{path}: protocol", + ) + require( + protocol.get("selection_split") == "development" + and protocol.get("confirmation_seeds_touched") is False, + f"{path}: split", + ) + require(protocol.get("grid_size") == 24, f"{path}: grid size") + require( + protocol.get("protocol_sha256") == digests["protocol"] + and protocol.get("d4_gate_sha256") == digests["d4_gate"] + and protocol.get("old_r2_gate_sha256") + == digests["old_r2_gate"], + f"{path}: parent digest", + ) + provenance = row.get("provenance", {}) + require( + provenance.get("git_tracked_dirty") is False, + f"{path}: dirty source", + ) + require( + provenance.get("tracked_inputs") + and all(provenance["tracked_inputs"].values()) + and set(provenance["tracked_inputs"]) == set(digests), + f"{path}: tracked inputs", + ) + require( + provenance.get("input_sha256") == digests, + f"{path}: source digest drift", + ) + require( + isinstance(provenance.get("git_commit"), str) + and len(provenance["git_commit"]) == 40, + f"{path}: source commit", + ) + expected_cfg = vars(build_config(*candidate)) + require(row.get("config") == expected_cfg, f"{path}: config") + require(row.get("finite") is True, f"{path}: finite") + require(row.get("split") == "development", f"{path}: cell split") + require( + set(row.get("conditions", {})) == set(CONDITIONS), + f"{path}: conditions", + ) + for name in CONDITIONS: + warmup = row["warmup"][name] + require( + warmup["batches"] == 100 + and warmup["examples"] == 6400 + and warmup["instruction_present"] is False + and warmup["role_cursor_scalar_observations"] == 12800 + and warmup["predictor_max_abs_error"] <= 1e-5, + f"{path}: warmup {name}", + ) + condition = row["conditions"][name] + require( + len(condition["daily_success"]) == 14, + f"{path}: daily trajectory {name}", + ) + cost = condition["cost"] + require( + cost["maximum_state_episode_steps"] == 25088 + and cost["cursor_scalar_observations"] + == 2 * cost["role_probe_examples"] + and cost["terminal_outcome_observations"] == 896 + and cost["task_loss_queries"] == 0 + and cost["reverse_mode_calls"] == 0, + f"{path}: cost {name}", + ) + challenge = row["assays"]["challenge"] + require( + challenge["horizons"] == list(HORIZONS) + and challenge["episodes_per_horizon"] == CHALLENGE_EPISODES + and challenge["selection_over_horizons"] is False, + f"{path}: challenge ladder", + ) + signatures = row["signatures"] + require( + signatures["challenge_episodes"] + == len(HORIZONS) * CHALLENGE_EPISODES, + f"{path}: challenge count", + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--results", default="results/bci_v2_dev") + parser.add_argument( + "--out", default="results/bci_v2_dev_gate.json" + ) + args = parser.parse_args() + paths = source_paths() + digests = {name: sha256(path) for name, path in paths.items()} + require( + os.path.abspath(paths["runner"]) == os.path.abspath(RUNNER_PATH), + "runner path", + ) + candidates = [ + (forward_eta, gamma, critic_eta) + for forward_eta in FORWARD_ETAS + for gamma in GAMMAS + for critic_eta in CRITIC_ETAS + ] + expected_names = { + filename(candidate, task_seed) + for candidate in candidates + for task_seed in TASK_SEEDS + } + observed_names = { + os.path.basename(path) + for path in glob.glob(os.path.join(args.results, "*.json")) + } + require( + observed_names == expected_names, + ( + "development grid drift: " + f"missing={sorted(expected_names - observed_names)}, " + f"extra={sorted(observed_names - expected_names)}" + ), + ) + records = {} + commits = set() + source_sha256 = {} + for candidate in candidates: + for task_seed in TASK_SEEDS: + path = os.path.join( + args.results, filename(candidate, task_seed) + ) + with open(path) as handle: + row = json.load(handle) + validate(row, path, candidate, task_seed, digests) + records[(candidate, task_seed)] = row + commits.add(row["provenance"]["git_commit"]) + source_sha256[path] = sha256(path) + require(len(commits) == 1, "development source revision drift") + + summaries = [] + eligible = [] + for candidate in candidates: + rows = [records[(candidate, seed)] for seed in TASK_SEEDS] + checks = { + str(seed): candidate_checks(row) + for seed, row in zip(TASK_SEEDS, rows) + } + passed = all( + value for seed_checks in checks.values() + for value in seed_checks.values() + ) + final_values = [ + row["conditions"]["intact"]["final_success"] + for row in rows + ] + outcome_values = [ + row["signatures"][ + "terminal_residual_outcome_balanced_acc" + ] + for row in rows + ] + summary = { + "forward_eta": candidate[0], + "gamma": candidate[1], + "critic_eta": candidate[2], + "eligible": passed, + "checks_by_task_seed": checks, + "intact_final_by_task_seed": final_values, + "terminal_outcome_accuracy_by_task_seed": outcome_values, + "worst_intact_final": min(final_values), + "mean_intact_final": statistics.mean(final_values), + "worst_terminal_outcome_accuracy": min(outcome_values), + } + summaries.append(summary) + if passed: + eligible.append(summary) + selected = None + if eligible: + selected = sorted( + eligible, + key=lambda row: ( + -row["worst_intact_final"], + -row["worst_terminal_outcome_accuracy"], + row["forward_eta"], + row["critic_eta"], + row["gamma"], + ), + )[0] + output = { + "protocol": "oral_b_v2_development_v1", + "status": "passed" if selected else "failed", + "complete_grid": True, + "grid_size": len(expected_names), + "candidate_summaries": summaries, + "selected": ( + { + "forward_eta": selected["forward_eta"], + "gamma": selected["gamma"], + "critic_eta": selected["critic_eta"], + "worst_intact_final": selected[ + "worst_intact_final" + ], + "worst_terminal_outcome_accuracy": selected[ + "worst_terminal_outcome_accuracy" + ], + } + if selected else None + ), + "confirmation_seeds_touched": False, + "oral_b_v2_confirmation_opened": selected is not None, + "source_commit": next(iter(commits)), + "source_sha256": source_sha256, + "input_sha256": digests, + "review_score_before": 7, + "review_score_after": 7, + "score_change_rule": ( + "development selection never changes the formal milestone score" + ), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + if os.path.exists(args.out): + with open(args.out) as handle: + existing = json.load(handle) + require( + existing == output, + "existing v2 development gate differs from deterministic audit", + ) + else: + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() |