diff options
| author | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 07:58:09 -0500 |
|---|---|---|
| committer | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 07:58:09 -0500 |
| commit | 670c1399659f31ac79c4b454147417d4f8ddca7d (patch) | |
| tree | aa14a4f1222154631900805c7f2219a13821db77 /experiments | |
| parent | ce10335dcc3aa12f4f1e6dd53352d7e71d0cccf7 (diff) | |
protocol: freeze independent oral-B-v2 gates
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_bci_v2_confirmation.py | 520 | ||||
| -rw-r--r-- | experiments/analyze_bci_v2_development.py | 370 | ||||
| -rw-r--r-- | experiments/bci_v2_confirmation.py | 149 | ||||
| -rw-r--r-- | experiments/bci_v2_confirmation.sh | 16 | ||||
| -rw-r--r-- | experiments/bci_v2_development_screen.sh | 22 | ||||
| -rw-r--r-- | experiments/bci_v2_run.py | 546 |
6 files changed, 1623 insertions, 0 deletions
diff --git a/experiments/analyze_bci_v2_confirmation.py b/experiments/analyze_bci_v2_confirmation.py new file mode 100644 index 0000000..36b59cf --- /dev/null +++ b/experiments/analyze_bci_v2_confirmation.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +"""Audit the frozen untouched oral-B-v2 confirmation panel.""" +import argparse +import glob +import hashlib +import json +import math +import os +import statistics + +from experiments.bci_v2_confirmation import ( + MODEL_SEEDS, + R1_GATE_PATH, + TASK_SEEDS, +) +from experiments.bci_v2_run import ( + CHALLENGE_EPISODES, + CONDITIONS, + HORIZONS, + build_config, + source_paths, +) + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RUNNER_PATH = os.path.join( + ROOT, "experiments", "bci_v2_confirmation.py" +) +T_CRITICAL_ONE_SIDED_95_DF5 = 2.015048373 + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def finite_tree(value): + if isinstance(value, dict): + return all(finite_tree(item) for item in value.values()) + if isinstance(value, (list, tuple)): + return all(finite_tree(item) for item in value) + if isinstance(value, (int, float)): + return math.isfinite(value) + return True + + +def lower_confidence_bound(values): + return ( + statistics.mean(values) + - T_CRITICAL_ONE_SIDED_95_DF5 + * statistics.stdev(values) + / math.sqrt(len(values)) + ) + + +def upper_confidence_bound(values): + return ( + statistics.mean(values) + + T_CRITICAL_ONE_SIDED_95_DF5 + * statistics.stdev(values) + / math.sqrt(len(values)) + ) + + +def summarize(values): + return { + "by_task_seed": values, + "mean": statistics.mean(values), + "one_sided_95pct_lower": lower_confidence_bound(values), + "one_sided_95pct_upper": upper_confidence_bound(values), + } + + +def task_cluster_values(records, getter): + return [ + statistics.mean( + getter(records[(task_seed, model_seed)]) + for model_seed in MODEL_SEEDS + ) + for task_seed in TASK_SEEDS + ] + + +def confirmation_source_paths(r1_gate_path): + paths = source_paths() + paths["development_runner"] = paths["runner"] + paths["runner"] = RUNNER_PATH + paths["r1_gate"] = os.path.abspath(r1_gate_path) + return paths + + +def validate(row, path, selected, digests): + require(row.get("schema_version") == 2, f"{path}: schema") + args = row.get("args", {}) + task_seed = args.get("task_seed") + model_seed = args.get("model_seed") + require(task_seed in TASK_SEEDS, f"{path}: task seed") + require(model_seed in MODEL_SEEDS, f"{path}: model seed") + protocol = row.get("protocol", {}) + require( + protocol.get("name") == "oral_b_v2_confirmation_v1" + and protocol.get("split") == "untouched_confirmation", + f"{path}: protocol", + ) + require( + protocol.get("training_task_seed") == task_seed + and protocol.get("model_seed") == model_seed, + f"{path}: seed binding", + ) + require( + protocol.get("selected") == selected + and protocol.get("no_further_selection") is True + and protocol.get("confirmation_grid_size") == 30, + f"{path}: selection", + ) + require( + protocol.get("protocol_sha256") == digests["protocol"] + and protocol.get("d4_gate_sha256") == digests["d4_gate"] + and protocol.get("old_r2_gate_sha256") + == digests["old_r2_gate"] + and protocol.get("r1_gate_sha256") == digests["r1_gate"], + f"{path}: gate digest", + ) + provenance = row.get("provenance", {}) + require( + provenance.get("git_tracked_dirty") is False, + f"{path}: dirty source", + ) + require( + provenance.get("tracked_inputs") + and all(provenance["tracked_inputs"].values()) + and set(provenance["tracked_inputs"]) == set(digests), + f"{path}: tracked inputs", + ) + require( + provenance.get("input_sha256") == digests, + f"{path}: source digest drift", + ) + expected_cfg = vars(build_config( + selected["forward_eta"], + selected["gamma"], + selected["critic_eta"], + )) + require(row.get("config") == expected_cfg, f"{path}: config") + require( + row.get("split") == "untouched_confirmation" + and row.get("finite") is True + and finite_tree(row), + f"{path}: finite/split", + ) + require( + set(row.get("conditions", {})) == set(CONDITIONS), + f"{path}: conditions", + ) + for name in CONDITIONS: + warmup = row["warmup"][name] + require( + warmup["batches"] == 100 + and warmup["examples"] == 6400 + and warmup["instruction_present"] is False + and warmup["role_cursor_scalar_observations"] == 12800 + and warmup["predictor_max_abs_error"] <= 1e-5, + f"{path}: warmup {name}", + ) + condition = row["conditions"][name] + require( + len(condition["daily_success"]) == 14, + f"{path}: trajectory {name}", + ) + cost = condition["cost"] + require( + cost["maximum_state_episode_steps"] == 25088 + and cost["cursor_scalar_observations"] + == 2 * cost["role_probe_examples"] + and cost["terminal_outcome_observations"] == 896 + and cost["task_loss_queries"] == 0 + and cost["reverse_mode_calls"] == 0, + f"{path}: cost {name}", + ) + challenge = row["assays"]["challenge"] + require( + challenge["horizons"] == list(HORIZONS) + and challenge["episodes_per_horizon"] == CHALLENGE_EPISODES + and challenge["selection_over_horizons"] is False, + f"{path}: challenge", + ) + require( + row["signatures"]["challenge_episodes"] + == len(HORIZONS) * CHALLENGE_EPISODES, + f"{path}: challenge count", + ) + + +def r2_checks(metrics, clustered, all_values): + return { + "all_records_finite_paired_and_cost_audited": True, + "learning_and_plasticity": { + "mean_intact_final_at_least_0p70": + metrics["intact_final"]["mean"] >= 0.70, + "every_task_mean_final_at_least_0p60": + min(clustered["intact_final"]) >= 0.60, + "intact_final_lower_bound_at_least_0p60": + metrics["intact_final"][ + "one_sided_95pct_lower" + ] >= 0.60, + "mean_learning_gain_at_least_0p10": + metrics["intact_gain"]["mean"] >= 0.10, + "learning_gain_lower_bound_at_least_0p05": + metrics["intact_gain"][ + "one_sided_95pct_lower" + ] >= 0.05, + "mean_fixed_role_gap_at_least_0p20": + metrics["fixed_role_gap"]["mean"] >= 0.20, + "fixed_role_gap_lower_bound_at_least_0p10": + metrics["fixed_role_gap"][ + "one_sided_95pct_lower" + ] >= 0.10, + "mean_oracle_deficit_at_most_0p10": + metrics["oracle_deficit"]["mean"] <= 0.10, + "oracle_deficit_upper_bound_at_most_0p20": + metrics["oracle_deficit"][ + "one_sided_95pct_upper" + ] <= 0.20, + "plasticity_half_margin_lower_bound_nonnegative": + metrics["plasticity_half_margin"][ + "one_sided_95pct_lower" + ] >= 0.0, + "mean_role_cosine_at_least_0p80": + metrics["role_cosine"]["mean"] >= 0.80, + "every_role_cosine_at_least_0p70": + min(all_values["role_cosine"]) >= 0.70, + }, + "innovation_and_network_prediction": { + "mean_residual_soma_corr_at_most_0p10": + metrics["residual_soma_corr"]["mean"] <= 0.10, + "residual_soma_corr_upper_bound_at_most_0p12": + metrics["residual_soma_corr"][ + "one_sided_95pct_upper" + ] <= 0.12, + "mean_raw_residual_corr_gap_at_least_0p20": + metrics["raw_residual_corr_gap"]["mean"] >= 0.20, + "raw_residual_corr_gap_lower_bound_at_least_0p15": + metrics["raw_residual_corr_gap"][ + "one_sided_95pct_lower" + ] >= 0.15, + "mean_surrounding_event_accuracy_at_least_0p52": + metrics["surrounding_event_accuracy"]["mean"] >= 0.52, + "surrounding_event_accuracy_lower_bound_at_least_0p50": + metrics["surrounding_event_accuracy"][ + "one_sided_95pct_lower" + ] >= 0.50, + "mean_decoder_distance_corr_at_least_0p05": + metrics["decoder_distance_corr"]["mean"] >= 0.05, + "decoder_distance_corr_lower_bound_nonnegative": + metrics["decoder_distance_corr"][ + "one_sided_95pct_lower" + ] >= 0.0, + "positive_sign_in_at_least_25_of_30": + sum( + value > 0 for value in all_values["sign_inversion"] + ) >= 25, + "positive_sign_in_every_task_cluster": + min(clustered["sign_inversion"]) > 0.0, + "mean_velocity_advantage_at_least_0p05": + metrics["velocity_advantage"]["mean"] >= 0.05, + "velocity_advantage_lower_bound_nonnegative": + metrics["velocity_advantage"][ + "one_sided_95pct_lower" + ] >= 0.0, + }, + "terminal_outcome_surprise": { + "every_challenge_fraction_between_0p10_and_0p90": + min(all_values["challenge_fraction"]) >= 0.10 + and max(all_values["challenge_fraction"]) <= 0.90, + "mean_terminal_outcome_accuracy_at_least_0p65": + metrics["terminal_outcome_accuracy"]["mean"] >= 0.65, + "terminal_outcome_accuracy_lower_bound_at_least_0p60": + metrics["terminal_outcome_accuracy"][ + "one_sided_95pct_lower" + ] >= 0.60, + "mean_terminal_role_separation_at_least_0p03": + metrics["terminal_role_separation"]["mean"] >= 0.03, + "terminal_role_separation_lower_bound_at_least_0p01": + metrics["terminal_role_separation"][ + "one_sided_95pct_lower" + ] >= 0.01, + "mean_acute_outcome_lesion_drop_at_least_0p01": + metrics["outcome_lesion_drop"]["mean"] >= 0.01, + "acute_outcome_lesion_drop_lower_bound_nonnegative": + metrics["outcome_lesion_drop"][ + "one_sided_95pct_lower" + ] >= 0.0, + "mean_critic_expectedness_at_least_0p005": + metrics["critic_expectedness"]["mean"] >= 0.005, + "critic_expectedness_lower_bound_nonnegative": + metrics["critic_expectedness"][ + "one_sided_95pct_lower" + ] >= 0.0, + "every_critic_contribution_value_corr_at_least_0p95": + min(all_values["critic_value_corr"]) >= 0.95, + }, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--results", default="results/bci_v2_confirmation" + ) + parser.add_argument( + "--r1-gate", default="results/bci_v2_dev_gate.json" + ) + parser.add_argument( + "--out", default="results/bci_v2_confirmation_gate.json" + ) + args = parser.parse_args() + with open(args.r1_gate) as handle: + r1 = json.load(handle) + require( + r1.get("protocol") == "oral_b_v2_development_v1" + and r1.get("status") == "passed" + and r1.get("complete_grid") is True + and r1.get("confirmation_seeds_touched") is False + and r1.get("oral_b_v2_confirmation_opened") is True, + "R1 gate", + ) + selected = { + key: r1["selected"][key] + for key in ("forward_eta", "gamma", "critic_eta") + } + paths = confirmation_source_paths(args.r1_gate) + digests = {name: sha256(path) for name, path in paths.items()} + development_digests = { + name: sha256(path) + for name, path in source_paths().items() + } + require( + r1.get("input_sha256") == development_digests, + "R1 source binding", + ) + expected_names = { + f"bci_v2_confirm_t{task_seed}_m{model_seed}.json" + for task_seed in TASK_SEEDS + for model_seed in MODEL_SEEDS + } + observed_names = { + os.path.basename(path) + for path in glob.glob(os.path.join(args.results, "*.json")) + } + require( + observed_names == expected_names, + ( + "confirmation grid drift: " + f"missing={sorted(expected_names - observed_names)}, " + f"extra={sorted(observed_names - expected_names)}" + ), + ) + records = {} + commits = set() + source_sha256 = {} + for task_seed in TASK_SEEDS: + for model_seed in MODEL_SEEDS: + path = os.path.join( + args.results, + f"bci_v2_confirm_t{task_seed}_m{model_seed}.json", + ) + with open(path) as handle: + row = json.load(handle) + validate(row, path, selected, digests) + records[(task_seed, model_seed)] = row + commits.add(row["provenance"]["git_commit"]) + source_sha256[path] = sha256(path) + require(len(commits) == 1, "confirmation source revision drift") + + getters = { + "intact_final": lambda row: + row["conditions"]["intact"]["final_success"], + "intact_gain": lambda row: + row["conditions"]["intact"]["learning_gain"], + "fixed_role_gap": lambda row: ( + row["conditions"]["intact"]["final_success"] + - row["conditions"]["fixed_role"]["final_success"] + ), + "oracle_deficit": lambda row: ( + row["conditions"]["oracle_role"]["final_success"] + - row["conditions"]["intact"]["final_success"] + ), + "plasticity_half_margin": lambda row: ( + 0.5 * row["conditions"]["intact"]["learning_gain"] + - row["conditions"]["plasticity_lesion"]["learning_gain"] + ), + "role_cosine": lambda row: + row["conditions"]["intact"]["role_cosine_after_training"], + "critic_training_gap": lambda row: ( + row["conditions"]["intact"]["final_success"] + - row["conditions"]["critic_training_lesion"][ + "final_success" + ] + ), + "outcome_training_gap": lambda row: ( + row["conditions"]["intact"]["final_success"] + - row["conditions"]["outcome_training_lesion"][ + "final_success" + ] + ), + "residual_soma_corr": lambda row: + row["signatures"]["mean_abs_residual_soma_corr"], + "raw_residual_corr_gap": lambda row: + row["signatures"][ + "raw_minus_residual_abs_soma_corr" + ], + "surrounding_event_accuracy": lambda row: + row["signatures"][ + "surrounding_event_decoder_balanced_acc" + ], + "decoder_distance_corr": lambda row: + row["signatures"]["decoder_distance_residual_corr"], + "sign_inversion": lambda row: + row["signatures"][ + "causal_role_sign_inversion_index" + ], + "velocity_advantage": lambda row: + row["signatures"][ + "velocity_minus_error_abs_cv_corr" + ], + "challenge_fraction": lambda row: + row["signatures"]["challenge_success_fraction"], + "terminal_outcome_accuracy": lambda row: + row["signatures"][ + "terminal_residual_outcome_balanced_acc" + ], + "terminal_role_separation": lambda row: + row["signatures"][ + "terminal_role_aligned_outcome_separation" + ], + "outcome_lesion_drop": lambda row: + row["signatures"][ + "terminal_outcome_separation_drop_under_acute_lesion" + ], + "critic_expectedness": lambda row: + row["signatures"][ + "mean_critic_expectedness_contribution" + ], + "critic_value_corr": lambda row: + row["signatures"][ + "critic_contribution_value_prediction_corr" + ], + } + clustered = { + name: task_cluster_values(records, getter) + for name, getter in getters.items() + } + metrics = { + name: summarize(values) + for name, values in clustered.items() + } + all_values = { + name: [getter(row) for row in records.values()] + for name, getter in getters.items() + } + checks = r2_checks(metrics, clustered, all_values) + passed = all( + value + for category in checks.values() + for value in ( + [category] + if isinstance(category, bool) + else category.values() + ) + ) + output = { + "protocol": "oral_b_v2_confirmation_v1", + "status": "passed" if passed else "failed", + "complete_grid": True, + "selected": selected, + "checks": checks, + "metrics": metrics, + "positive_sign_count": sum( + value > 0 for value in all_values["sign_inversion"] + ), + "source_commit": next(iter(commits)), + "source_sha256": source_sha256, + "input_sha256": digests, + "oral_b_v2_outcome_surprise_established": passed, + "old_r2_status_preserved": "failed", + "old_oral_a_gate_remains_closed": True, + "new_oral_a_v2_protocol_may_be_frozen": passed, + "review_score_before": 7, + "review_score_after": 8 if passed else 7, + "score_change_rule": ( + "only a complete untouched v2 R2 pass establishes " + "role-vectorized TD outcome surprise; the failed old R2 " + "and its old oral-A gate remain unchanged" + ), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + if os.path.exists(args.out): + with open(args.out) as handle: + existing = json.load(handle) + require( + existing == output, + "existing v2 R2 gate differs from deterministic re-audit", + ) + else: + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/analyze_bci_v2_development.py b/experiments/analyze_bci_v2_development.py new file mode 100644 index 0000000..4acf321 --- /dev/null +++ b/experiments/analyze_bci_v2_development.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 +"""Audit and select the frozen oral-B-v2 development grid.""" +import argparse +import glob +import hashlib +import json +import os +import statistics + +from experiments.bci_v2_run import ( + CHALLENGE_EPISODES, + CONDITIONS, + CRITIC_ETAS, + FORWARD_ETAS, + GAMMAS, + HORIZONS, + MODEL_SEEDS, + PROTOCOL_PATH, + TASK_SEEDS, + build_config, + source_paths, +) + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +RUNNER_PATH = os.path.join(ROOT, "experiments", "bci_v2_run.py") + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def tag(value): + return str(value).replace(".", "p") + + +def filename(candidate, task_seed): + forward_eta, gamma, critic_eta = candidate + return ( + f"bci_v2_e{tag(forward_eta)}_g{tag(gamma)}" + f"_c{tag(critic_eta)}_t{task_seed}_m0.json" + ) + + +def candidate_checks(row): + conditions = row["conditions"] + signatures = row["signatures"] + intact = conditions["intact"] + return { + "final_success_at_least_0p70": + intact["final_success"] >= 0.70, + "learning_gain_at_least_0p10": + intact["learning_gain"] >= 0.10, + "fixed_role_final_gap_at_least_0p20": ( + intact["final_success"] + - conditions["fixed_role"]["final_success"] + ) >= 0.20, + "oracle_final_deficit_at_most_0p10": ( + conditions["oracle_role"]["final_success"] + - intact["final_success"] + ) <= 0.10, + "plasticity_lesion_retains_at_most_half_gain": + conditions["plasticity_lesion"]["learning_gain"] + <= 0.5 * intact["learning_gain"], + "learned_role_cosine_at_least_0p80": + intact["role_cosine_after_training"] >= 0.80, + "challenge_success_fraction_between_0p15_and_0p85": + 0.15 <= signatures["challenge_success_fraction"] <= 0.85, + "terminal_outcome_accuracy_at_least_0p65": + signatures[ + "terminal_residual_outcome_balanced_acc" + ] >= 0.65, + "terminal_role_separation_at_least_0p03": + signatures[ + "terminal_role_aligned_outcome_separation" + ] >= 0.03, + "acute_outcome_lesion_separation_drop_at_least_0p01": + signatures[ + "terminal_outcome_separation_drop_under_acute_lesion" + ] >= 0.01, + "critic_expectedness_contribution_at_least_0p005": + signatures[ + "mean_critic_expectedness_contribution" + ] >= 0.005, + "critic_contribution_tracks_value_at_least_0p95": + signatures[ + "critic_contribution_value_prediction_corr" + ] >= 0.95, + "residual_soma_corr_at_most_0p10": + signatures["mean_abs_residual_soma_corr"] <= 0.10, + "raw_residual_corr_gap_at_least_0p20": + signatures[ + "raw_minus_residual_abs_soma_corr" + ] >= 0.20, + "surrounding_event_accuracy_at_least_0p52": + signatures[ + "surrounding_event_decoder_balanced_acc" + ] >= 0.52, + "decoder_distance_corr_at_least_0p02": + signatures[ + "decoder_distance_residual_corr" + ] >= 0.02, + "sign_inversion_at_least_0p01": + signatures[ + "causal_role_sign_inversion_index" + ] >= 0.01, + "velocity_advantage_at_least_0p05": + signatures[ + "velocity_minus_error_abs_cv_corr" + ] >= 0.05, + } + + +def validate(row, path, candidate, task_seed, digests): + require(row.get("schema_version") == 2, f"{path}: schema") + args = row.get("args", {}) + require(args.get("task_seed") == task_seed, f"{path}: task seed") + require(args.get("model_seed") in MODEL_SEEDS, f"{path}: model seed") + require( + ( + args.get("forward_eta"), + args.get("gamma"), + args.get("critic_eta"), + ) == candidate, + f"{path}: candidate", + ) + protocol = row.get("protocol", {}) + require( + protocol.get("name") == "oral_b_v2_development_v1", + f"{path}: protocol", + ) + require( + protocol.get("selection_split") == "development" + and protocol.get("confirmation_seeds_touched") is False, + f"{path}: split", + ) + require(protocol.get("grid_size") == 24, f"{path}: grid size") + require( + protocol.get("protocol_sha256") == digests["protocol"] + and protocol.get("d4_gate_sha256") == digests["d4_gate"] + and protocol.get("old_r2_gate_sha256") + == digests["old_r2_gate"], + f"{path}: parent digest", + ) + provenance = row.get("provenance", {}) + require( + provenance.get("git_tracked_dirty") is False, + f"{path}: dirty source", + ) + require( + provenance.get("tracked_inputs") + and all(provenance["tracked_inputs"].values()) + and set(provenance["tracked_inputs"]) == set(digests), + f"{path}: tracked inputs", + ) + require( + provenance.get("input_sha256") == digests, + f"{path}: source digest drift", + ) + require( + isinstance(provenance.get("git_commit"), str) + and len(provenance["git_commit"]) == 40, + f"{path}: source commit", + ) + expected_cfg = vars(build_config(*candidate)) + require(row.get("config") == expected_cfg, f"{path}: config") + require(row.get("finite") is True, f"{path}: finite") + require(row.get("split") == "development", f"{path}: cell split") + require( + set(row.get("conditions", {})) == set(CONDITIONS), + f"{path}: conditions", + ) + for name in CONDITIONS: + warmup = row["warmup"][name] + require( + warmup["batches"] == 100 + and warmup["examples"] == 6400 + and warmup["instruction_present"] is False + and warmup["role_cursor_scalar_observations"] == 12800 + and warmup["predictor_max_abs_error"] <= 1e-5, + f"{path}: warmup {name}", + ) + condition = row["conditions"][name] + require( + len(condition["daily_success"]) == 14, + f"{path}: daily trajectory {name}", + ) + cost = condition["cost"] + require( + cost["maximum_state_episode_steps"] == 25088 + and cost["cursor_scalar_observations"] + == 2 * cost["role_probe_examples"] + and cost["terminal_outcome_observations"] == 896 + and cost["task_loss_queries"] == 0 + and cost["reverse_mode_calls"] == 0, + f"{path}: cost {name}", + ) + challenge = row["assays"]["challenge"] + require( + challenge["horizons"] == list(HORIZONS) + and challenge["episodes_per_horizon"] == CHALLENGE_EPISODES + and challenge["selection_over_horizons"] is False, + f"{path}: challenge ladder", + ) + signatures = row["signatures"] + require( + signatures["challenge_episodes"] + == len(HORIZONS) * CHALLENGE_EPISODES, + f"{path}: challenge count", + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--results", default="results/bci_v2_dev") + parser.add_argument( + "--out", default="results/bci_v2_dev_gate.json" + ) + args = parser.parse_args() + paths = source_paths() + digests = {name: sha256(path) for name, path in paths.items()} + require( + os.path.abspath(paths["runner"]) == os.path.abspath(RUNNER_PATH), + "runner path", + ) + candidates = [ + (forward_eta, gamma, critic_eta) + for forward_eta in FORWARD_ETAS + for gamma in GAMMAS + for critic_eta in CRITIC_ETAS + ] + expected_names = { + filename(candidate, task_seed) + for candidate in candidates + for task_seed in TASK_SEEDS + } + observed_names = { + os.path.basename(path) + for path in glob.glob(os.path.join(args.results, "*.json")) + } + require( + observed_names == expected_names, + ( + "development grid drift: " + f"missing={sorted(expected_names - observed_names)}, " + f"extra={sorted(observed_names - expected_names)}" + ), + ) + records = {} + commits = set() + source_sha256 = {} + for candidate in candidates: + for task_seed in TASK_SEEDS: + path = os.path.join( + args.results, filename(candidate, task_seed) + ) + with open(path) as handle: + row = json.load(handle) + validate(row, path, candidate, task_seed, digests) + records[(candidate, task_seed)] = row + commits.add(row["provenance"]["git_commit"]) + source_sha256[path] = sha256(path) + require(len(commits) == 1, "development source revision drift") + + summaries = [] + eligible = [] + for candidate in candidates: + rows = [records[(candidate, seed)] for seed in TASK_SEEDS] + checks = { + str(seed): candidate_checks(row) + for seed, row in zip(TASK_SEEDS, rows) + } + passed = all( + value for seed_checks in checks.values() + for value in seed_checks.values() + ) + final_values = [ + row["conditions"]["intact"]["final_success"] + for row in rows + ] + outcome_values = [ + row["signatures"][ + "terminal_residual_outcome_balanced_acc" + ] + for row in rows + ] + summary = { + "forward_eta": candidate[0], + "gamma": candidate[1], + "critic_eta": candidate[2], + "eligible": passed, + "checks_by_task_seed": checks, + "intact_final_by_task_seed": final_values, + "terminal_outcome_accuracy_by_task_seed": outcome_values, + "worst_intact_final": min(final_values), + "mean_intact_final": statistics.mean(final_values), + "worst_terminal_outcome_accuracy": min(outcome_values), + } + summaries.append(summary) + if passed: + eligible.append(summary) + selected = None + if eligible: + selected = sorted( + eligible, + key=lambda row: ( + -row["worst_intact_final"], + -row["worst_terminal_outcome_accuracy"], + row["forward_eta"], + row["critic_eta"], + row["gamma"], + ), + )[0] + output = { + "protocol": "oral_b_v2_development_v1", + "status": "passed" if selected else "failed", + "complete_grid": True, + "grid_size": len(expected_names), + "candidate_summaries": summaries, + "selected": ( + { + "forward_eta": selected["forward_eta"], + "gamma": selected["gamma"], + "critic_eta": selected["critic_eta"], + "worst_intact_final": selected[ + "worst_intact_final" + ], + "worst_terminal_outcome_accuracy": selected[ + "worst_terminal_outcome_accuracy" + ], + } + if selected else None + ), + "confirmation_seeds_touched": False, + "oral_b_v2_confirmation_opened": selected is not None, + "source_commit": next(iter(commits)), + "source_sha256": source_sha256, + "input_sha256": digests, + "review_score_before": 7, + "review_score_after": 7, + "score_change_rule": ( + "development selection never changes the formal milestone score" + ), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + if os.path.exists(args.out): + with open(args.out) as handle: + existing = json.load(handle) + require( + existing == output, + "existing v2 development gate differs from deterministic audit", + ) + else: + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/bci_v2_confirmation.py b/experiments/bci_v2_confirmation.py new file mode 100644 index 0000000..6968ee7 --- /dev/null +++ b/experiments/bci_v2_confirmation.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Run one untouched oral-B-v2 confirmation cell.""" +import argparse +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from experiments.bci_v2_run import ( + D4_GATE_PATH, + OLD_R2_GATE_PATH, + PROTOCOL_PATH, + finite_tree, + provenance, + require_parent_gates, + run_cell, + sha256, + source_paths, +) + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +R1_GATE_PATH = os.path.join(ROOT, "results", "bci_v2_dev_gate.json") +TASK_SEEDS = tuple(range(30, 36)) +MODEL_SEEDS = tuple(range(5)) + + +def require_r1_gate(r1): + development_digests = { + name: sha256(path) + for name, path in source_paths().items() + } + selected = r1.get("selected") or {} + if not ( + r1.get("protocol") == "oral_b_v2_development_v1" + and r1.get("status") == "passed" + and r1.get("complete_grid") is True + and r1.get("grid_size") == 24 + and r1.get("confirmation_seeds_touched") is False + and r1.get("oral_b_v2_confirmation_opened") is True + and r1.get("review_score_after") == 7 + and r1.get("input_sha256") == development_digests + and selected.get("forward_eta") in (0.03, 0.1) + and selected.get("gamma") in (0.8, 0.95) + and selected.get("critic_eta") in (0.01, 0.03) + ): + raise ValueError( + "oral-B-v2 R2 requires the complete eligible frozen R1 gate" + ) + return selected + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--task-seed", type=int, choices=TASK_SEEDS, required=True + ) + parser.add_argument( + "--model-seed", type=int, choices=MODEL_SEEDS, required=True + ) + parser.add_argument( + "--r1-gate", default="results/bci_v2_dev_gate.json" + ) + parser.add_argument( + "--outdir", default="results/bci_v2_confirmation" + ) + args = parser.parse_args() + require_parent_gates() + with open(args.r1_gate) as handle: + r1 = json.load(handle) + selected = require_r1_gate(r1) + extra_paths = { + "development_runner": os.path.join( + ROOT, "experiments", "bci_v2_run.py" + ), + "runner": os.path.abspath(__file__), + "r1_gate": os.path.abspath(args.r1_gate), + } + source = provenance(extra_paths) + if ( + source["git_tracked_dirty"] + or not all(source["tracked_inputs"].values()) + ): + raise RuntimeError( + "oral-B-v2 R2 requires clean, tracked, frozen inputs" + ) + cell = run_cell( + args.task_seed, + args.model_seed, + selected["forward_eta"], + selected["gamma"], + selected["critic_eta"], + split="untouched_confirmation", + performance_seed_offset=500_000, + challenge_seed_offset=510_000, + ) + result = { + "schema_version": 2, + "protocol": { + "name": "oral_b_v2_confirmation_v1", + "split": "untouched_confirmation", + "training_task_seed": args.task_seed, + "model_seed": args.model_seed, + "selected": { + "forward_eta": selected["forward_eta"], + "gamma": selected["gamma"], + "critic_eta": selected["critic_eta"], + }, + "no_further_selection": True, + "confirmation_grid_size": ( + len(TASK_SEEDS) * len(MODEL_SEEDS) + ), + "protocol_sha256": sha256(PROTOCOL_PATH), + "d4_gate_sha256": sha256(D4_GATE_PATH), + "old_r2_gate_sha256": sha256(OLD_R2_GATE_PATH), + "r1_gate_sha256": sha256(args.r1_gate), + }, + "args": vars(args), + "provenance": source, + **cell, + } + result["finite"] = finite_tree(result) + if not result["finite"]: + raise RuntimeError("non-finite oral-B-v2 confirmation record") + os.makedirs(args.outdir, exist_ok=True) + path = os.path.join( + args.outdir, + ( + f"bci_v2_confirm_t{args.task_seed}" + f"_m{args.model_seed}.json" + ), + ) + if os.path.exists(path): + raise FileExistsError(f"refusing to overwrite {path}") + with open(path, "w") as handle: + json.dump(result, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "path": path, + "intact_final": result["conditions"]["intact"]["final_success"], + "terminal_outcome_accuracy": result["signatures"][ + "terminal_residual_outcome_balanced_acc" + ], + "finite": result["finite"], + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/bci_v2_confirmation.sh b/experiments/bci_v2_confirmation.sh new file mode 100644 index 0000000..da7328c --- /dev/null +++ b/experiments/bci_v2_confirmation.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Frozen oral-B-v2 R2: selected candidate x 6 task x 5 model seeds. +set -euo pipefail + +export OMP_NUM_THREADS=1 +export MKL_NUM_THREADS=1 + +for task_seed in 30 31 32 33 34 35; do + for model_seed in 0 1 2 3 4; do + python experiments/bci_v2_confirmation.py \ + --task-seed "$task_seed" \ + --model-seed "$model_seed" + done +done + +python experiments/analyze_bci_v2_confirmation.py diff --git a/experiments/bci_v2_development_screen.sh b/experiments/bci_v2_development_screen.sh new file mode 100644 index 0000000..8612cae --- /dev/null +++ b/experiments/bci_v2_development_screen.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Frozen oral-B-v2 R1: 8 candidates x 3 development task seeds. +set -euo pipefail + +export OMP_NUM_THREADS=1 +export MKL_NUM_THREADS=1 + +for forward_eta in 0.03 0.1; do + for gamma in 0.8 0.95; do + for critic_eta in 0.01 0.03; do + for task_seed in 20 21 22; do + python experiments/bci_v2_run.py \ + --forward-eta "$forward_eta" \ + --gamma "$gamma" \ + --critic-eta "$critic_eta" \ + --task-seed "$task_seed" + done + done + done +done + +python experiments/analyze_bci_v2_development.py diff --git a/experiments/bci_v2_run.py b/experiments/bci_v2_run.py new file mode 100644 index 0000000..5edebad --- /dev/null +++ b/experiments/bci_v2_run.py @@ -0,0 +1,546 @@ +#!/usr/bin/env python3 +"""Run one frozen oral-B-v2 development cell.""" +import argparse +import hashlib +import json +import math +import os +import platform +import resource +import subprocess +import sys +import time + +import torch + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from sdil.bci import generate_trajectories +from sdil.bci_v2 import BCIV2, BCIV2Config, run_day_v2 +from sdil.bci_v2_metrics import annotate_events, signature_metrics_v2 + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROTOCOL_PATH = os.path.join(ROOT, "ORAL_B_V2.md") +D4_GATE_PATH = os.path.join( + ROOT, "results", "kp_dynamic_projection_confirmation_gate.json" +) +OLD_R2_GATE_PATH = os.path.join( + ROOT, "results", "bci_td_confirmation_gate.json" +) +TASK_SEEDS = (20, 21, 22) +MODEL_SEEDS = (0,) +FORWARD_ETAS = (0.03, 0.1) +GAMMAS = (0.8, 0.95) +CRITIC_ETAS = (0.01, 0.03) +HORIZONS = (4, 8, 12, 16, 20, 24, 28) +CHALLENGE_EPISODES = 128 +CONDITIONS = ( + "intact", + "fixed_role", + "plasticity_lesion", + "critic_training_lesion", + "outcome_training_lesion", + "oracle_role", +) +ACUTE_MODES = ( + "intact", + "acute_critic_lesion", + "acute_outcome_lesion", +) + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def finite_tree(value): + if isinstance(value, dict): + return all(finite_tree(item) for item in value.values()) + if isinstance(value, (list, tuple)): + return all(finite_tree(item) for item in value) + if isinstance(value, (int, float)): + return math.isfinite(value) + return True + + +def source_paths(): + return { + "runner": os.path.abspath(__file__), + "development_analyzer": os.path.join( + ROOT, "experiments", "analyze_bci_v2_development.py" + ), + "confirmation_runner": os.path.join( + ROOT, "experiments", "bci_v2_confirmation.py" + ), + "confirmation_analyzer": os.path.join( + ROOT, "experiments", "analyze_bci_v2_confirmation.py" + ), + "base_dynamics": os.path.join(ROOT, "sdil", "bci.py"), + "v2_dynamics": os.path.join(ROOT, "sdil", "bci_v2.py"), + "v2_metrics": os.path.join(ROOT, "sdil", "bci_v2_metrics.py"), + "protocol": PROTOCOL_PATH, + "d4_gate": D4_GATE_PATH, + "old_r2_gate": OLD_R2_GATE_PATH, + } + + +def provenance(extra_paths=None): + def run(command): + return subprocess.run( + command, + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + paths = source_paths() + if extra_paths: + paths.update(extra_paths) + relative = { + name: os.path.relpath(path, ROOT) + for name, path in paths.items() + } + tracked = { + name: subprocess.run( + ["git", "ls-files", "--error-unmatch", path], + cwd=ROOT, + capture_output=True, + ).returncode == 0 + for name, path in relative.items() + } + return { + "git_commit": run(["git", "rev-parse", "HEAD"]), + "git_tracked_dirty": bool(run([ + "git", "status", "--porcelain", "--untracked-files=no" + ])), + "tracked_inputs": tracked, + "input_sha256": { + name: sha256(path) for name, path in paths.items() + }, + } + + +def require_parent_gates(): + with open(D4_GATE_PATH) as handle: + d4 = json.load(handle) + with open(OLD_R2_GATE_PATH) as handle: + old_r2 = json.load(handle) + if not ( + d4.get("protocol") + == "kp_dynamic_neutral_projection_confirmation_v1" + and d4.get("status") == "passed" + and d4.get("review_score_after") == 7 + ): + raise ValueError("oral-B-v2 requires the complete D4 accept gate") + if not ( + old_r2.get("protocol") == "oral_b_td_confirmation_v1" + and old_r2.get("status") == "failed" + and old_r2.get("review_score_after") == 7 + ): + raise ValueError("oral-B-v2 preserves and requires the failed old R2") + return d4, old_r2 + + +def build_config(forward_eta, gamma, critic_eta): + return BCIV2Config( + n_plus=5, + n_minus=5, + n_background=30, + context_dim=16, + steps_per_episode=28, + episodes_per_day=64, + days=14, + target=0.8, + inertia=0.65, + process_noise=0.12, + context_ar=0.8, + coupling_scale=1.0, + predictor_eta=0.2, + vectorizer_eta=0.03, + forward_eta=forward_eta, + perturb_sigma=0.03, + perturb_every=4, + kappa=0.0, + feedback="performance_velocity", + gamma=gamma, + critic_eta=critic_eta, + eligibility_decay=0.8, + velocity_reward_scale=0.25, + terminal_reward=1.0, + ) + + +def condition_settings(name): + return { + "intact": {}, + "fixed_role": {"learn_role": False}, + "plasticity_lesion": {"plasticity_gain": 0.0}, + "critic_training_lesion": { + "learn_critic": False, + "critic_enabled": False, + }, + "outcome_training_lesion": { + "terminal_outcome_enabled": False, + }, + "oracle_role": {"learn_role": False}, + }[name] + + +def neutral_warmup(initial, task_seed, model_seed, condition, batches=100): + """Fit neutral traffic and expose every condition to paired role probes.""" + model = initial.clone() + generator = torch.Generator(device="cpu").manual_seed( + 600_000 + 100 * task_seed + model_seed + ) + active = torch.ones(model.cfg.episodes_per_day, dtype=torch.bool) + update_role = condition not in ("fixed_role", "oracle_role") + scalar_observations = 0 + for _ in range(batches): + soma = torch.randn( + model.cfg.episodes_per_day, + model.cfg.n_neurons, + generator=generator, + ).tanh() + model.update_predictor( + soma, model.coupling * soma, active + ) + perturbations = torch.empty_like(soma).bernoulli_( + 0.5, generator=generator + ).mul_(2).sub_(1) + sigma = model.cfg.perturb_sigma + cursor_plus = (soma + sigma * perturbations) @ model.role + cursor_minus = (soma - sigma * perturbations) @ model.role + scalar_observations += 2 * soma.shape[0] + target = model.causal_role_targets( + cursor_plus, cursor_minus, perturbations + ) + if update_role: + model.update_role(target, active) + if condition == "oracle_role": + model.A.copy_(model.role) + return model, { + "batches": batches, + "examples": batches * model.cfg.episodes_per_day, + "instruction_present": False, + "role_cursor_scalar_observations": scalar_observations, + "role_update_enabled": update_role, + "predictor_max_abs_error": float( + (model.P - model.coupling).abs().max() + ), + "role_cosine_after_warmup": float( + torch.nn.functional.cosine_similarity( + model.A, model.role, dim=0 + ).item() + ), + "rng_seed": 600_000 + 100 * task_seed + model_seed, + } + + +def train(model, trajectories, condition, collect): + settings = condition_settings(condition) + daily_success = [] + events = [] + global_step = 0 + active_transitions = 0 + role_probe_examples = 0 + started = time.perf_counter() + for day in range(model.cfg.days): + report = run_day_v2( + model, + trajectories, + day, + global_step=global_step, + collect=collect, + learn_predictor=True, + probe_role=True, + **settings, + ) + global_step = report["global_step"] + active_transitions += report["active_transitions"] + role_probe_examples += report["role_probe_examples"] + daily_success.append(report["success_rate"]) + if collect: + annotate_events( + report["events"], + day, + report["success"], + day * model.cfg.episodes_per_day, + model.cfg.steps_per_episode, + ) + events.extend(report["events"]) + return events, { + "daily_success": daily_success, + "early_success": sum(daily_success[:3]) / 3, + "late_success": sum(daily_success[-3:]) / 3, + "learning_gain": ( + sum(daily_success[-3:]) + - sum(daily_success[:3]) + ) / 3, + "training_wall_s": time.perf_counter() - started, + "role_cosine_after_training": float( + torch.nn.functional.cosine_similarity( + model.A, model.role, dim=0 + ).item() + ), + "critic_l2_after_training": float(model.critic.norm().item()), + "cost": { + "maximum_state_episode_steps": ( + model.cfg.days + * model.cfg.episodes_per_day + * model.cfg.steps_per_episode + ), + "active_state_episode_steps": active_transitions, + "role_probe_examples": role_probe_examples, + "cursor_scalar_observations": 2 * role_probe_examples, + "terminal_outcome_observations": ( + model.cfg.days * model.cfg.episodes_per_day + ), + "task_loss_queries": 0, + "reverse_mode_calls": 0, + }, + } + + +def evaluate_performance(model, trajectories): + return run_day_v2( + model.clone(), + trajectories, + 0, + horizon=model.cfg.steps_per_episode, + plasticity_gain=0.0, + learn_role=False, + probe_role=False, + learn_predictor=False, + learn_critic=False, + collect=False, + ) + + +def evaluate_challenge_ladder(model, task_seed, seed_offset): + """Paired acute readouts over a fixed, non-selected horizon ladder.""" + mode_events = {name: [] for name in ACUTE_MODES} + costs = {name: 0 for name in ACUTE_MODES} + seeds = {} + for horizon_index, horizon in enumerate(HORIZONS): + trajectory_seed = ( + seed_offset + 1_000 * task_seed + horizon_index + ) + seeds[str(horizon)] = trajectory_seed + trajectories = generate_trajectories( + model.cfg, + trajectory_seed, + days=1, + episodes=CHALLENGE_EPISODES, + ) + for mode in ACUTE_MODES: + settings = { + "intact": {}, + "acute_critic_lesion": {"critic_enabled": False}, + "acute_outcome_lesion": { + "terminal_outcome_enabled": False + }, + }[mode] + report = run_day_v2( + model.clone(), + trajectories, + 0, + horizon=horizon, + plasticity_gain=0.0, + learn_role=False, + probe_role=False, + learn_predictor=False, + learn_critic=False, + collect=True, + **settings, + ) + episode_offset = ( + 1_000_000 + horizon_index * CHALLENGE_EPISODES + ) + annotate_events( + report["events"], + 0, + report["success"], + episode_offset, + horizon, + ) + mode_events[mode].extend(report["events"]) + costs[mode] += report["active_transitions"] + return mode_events, { + "horizons": list(HORIZONS), + "episodes_per_horizon": CHALLENGE_EPISODES, + "trajectory_seeds": seeds, + "active_state_episode_steps_by_mode": costs, + "selection_over_horizons": False, + } + + +def run_cell( + task_seed, + model_seed, + forward_eta, + gamma, + critic_eta, + *, + split, + performance_seed_offset, + challenge_seed_offset, +): + cfg = build_config(forward_eta, gamma, critic_eta) + trajectories = generate_trajectories(cfg, task_seed) + performance_seed = performance_seed_offset + task_seed + performance_trajectories = generate_trajectories( + cfg, performance_seed, days=1, episodes=256 + ) + initial = BCIV2(cfg, model_seed) + conditions = {} + trained = {} + warmups = {} + training_events = None + started = time.perf_counter() + for name in CONDITIONS: + model, warmup = neutral_warmup( + initial, task_seed, model_seed, name + ) + events, report = train( + model, trajectories, name, collect=name == "intact" + ) + warmups[name] = warmup + conditions[name] = report + trained[name] = model + if name == "intact": + training_events = events + for name in CONDITIONS: + report = evaluate_performance( + trained[name], performance_trajectories + ) + conditions[name]["final_success"] = report["success_rate"] + conditions[name]["evaluation_active_state_episode_steps"] = ( + report["active_transitions"] + ) + challenge_events, challenge_cost = evaluate_challenge_ladder( + trained["intact"], task_seed, challenge_seed_offset + ) + signatures = signature_metrics_v2( + training_events, + challenge_events, + cfg, + trained["intact"].role, + ) + return { + "config": vars(cfg), + "warmup": warmups, + "conditions": conditions, + "signatures": signatures, + "assays": { + "performance_evaluation_seed": performance_seed, + "performance_evaluation_episodes": 256, + "challenge": challenge_cost, + }, + "wall_s": time.perf_counter() - started, + "peak_rss_mib": ( + resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024 + ), + "hardware": { + "device": "cpu", + "platform": platform.platform(), + "torch_version": torch.__version__, + "threads": torch.get_num_threads(), + }, + "split": split, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--task-seed", type=int, choices=TASK_SEEDS, required=True + ) + parser.add_argument( + "--model-seed", type=int, choices=MODEL_SEEDS, default=0 + ) + parser.add_argument( + "--forward-eta", type=float, choices=FORWARD_ETAS, required=True + ) + parser.add_argument( + "--gamma", type=float, choices=GAMMAS, required=True + ) + parser.add_argument( + "--critic-eta", type=float, choices=CRITIC_ETAS, required=True + ) + parser.add_argument("--outdir", default="results/bci_v2_dev") + args = parser.parse_args() + require_parent_gates() + source = provenance() + if ( + source["git_tracked_dirty"] + or not all(source["tracked_inputs"].values()) + ): + raise RuntimeError( + "oral-B-v2 R1 requires clean, tracked, frozen inputs" + ) + cell = run_cell( + args.task_seed, + args.model_seed, + args.forward_eta, + args.gamma, + args.critic_eta, + split="development", + performance_seed_offset=400_000, + challenge_seed_offset=410_000, + ) + result = { + "schema_version": 2, + "protocol": { + "name": "oral_b_v2_development_v1", + "selection_split": "development", + "confirmation_seeds_touched": False, + "grid_size": ( + len(TASK_SEEDS) + * len(FORWARD_ETAS) + * len(GAMMAS) + * len(CRITIC_ETAS) + ), + "protocol_sha256": source["input_sha256"]["protocol"], + "d4_gate_sha256": source["input_sha256"]["d4_gate"], + "old_r2_gate_sha256": source["input_sha256"]["old_r2_gate"], + }, + "args": vars(args), + "provenance": source, + **cell, + } + result["finite"] = finite_tree(result) + if not result["finite"]: + raise RuntimeError("non-finite oral-B-v2 development record") + os.makedirs(args.outdir, exist_ok=True) + eta = str(args.forward_eta).replace(".", "p") + gamma = str(args.gamma).replace(".", "p") + critic = str(args.critic_eta).replace(".", "p") + path = os.path.join( + args.outdir, + ( + f"bci_v2_e{eta}_g{gamma}_c{critic}" + f"_t{args.task_seed}_m{args.model_seed}.json" + ), + ) + if os.path.exists(path): + raise FileExistsError(f"refusing to overwrite {path}") + with open(path, "w") as handle: + json.dump(result, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps({ + "path": path, + "intact_final": result["conditions"]["intact"]["final_success"], + "terminal_outcome_accuracy": result["signatures"][ + "terminal_residual_outcome_balanced_acc" + ], + "finite": result["finite"], + }, indent=2)) + + +if __name__ == "__main__": + main() |
