diff options
| author | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 08:28:44 -0500 |
|---|---|---|
| committer | YurenHao0426 <Blackhao0426@gmail.com> | 2026-07-23 08:28:44 -0500 |
| commit | 8f9364e8ee2e2a19eb83b58452fa0e8cb155353c (patch) | |
| tree | 5c883bfe32696317dc7f07e5ae71b24c6f40e332 | |
| parent | 6ebf1f1858590b53b47f67708cbd79aa57f79a67 (diff) | |
protocol: freeze BCI-v2-gated oral-A scaling
| -rw-r--r-- | ORAL_A_RECOVERY_V2.md | 103 | ||||
| -rw-r--r-- | experiments/analyze_oral_a_dynamic_scaling_v2.py | 256 | ||||
| -rw-r--r-- | experiments/oral_a_dynamic_scaling_v2.py | 149 | ||||
| -rw-r--r-- | experiments/oral_a_dynamic_scaling_v2_smoke.py | 73 |
4 files changed, 581 insertions, 0 deletions
diff --git a/ORAL_A_RECOVERY_V2.md b/ORAL_A_RECOVERY_V2.md new file mode 100644 index 0000000..e1aef12 --- /dev/null +++ b/ORAL_A_RECOVERY_V2.md @@ -0,0 +1,103 @@ +# Oral-A recovery v2: BCI-v2-authorized standard-depth scaling + +## Status and provenance boundary + +This is a new confirmation branch opened only by the complete calibrated +oral-B-v2 pass. It does not reopen or relabel the failed learned-vectorizer +Oral-A-v1--v4 branches, and it does not alter the closed legacy recovery whose +prerequisite was the failed original oral-B R2 gate. + +The experiment and every numerical threshold below are copied verbatim from +`ORAL_A_RECOVERY.md`, which was committed before any standard-depth endpoint +was generated. The only protocol change is the prerequisite: the new branch +requires both the complete D4 accept pass and +`oral_b_v2_calibrated_recovery_confirmation_v1`. No recovery-v2 test endpoint +may be generated until this document, its runner, analyzer, and smoke audit +are committed together in a clean tracked revision. + +This separation matters for interpretation. The calibrated BCI-v2 experiment +establishes that role-vectorized somato-dendritic innovation can carry local +TD/outcome surprise without Kolen--Pollack reciprocity. The panel below then +tests the complementary engineering claim that the already established +dynamic-innovation/reciprocal substrate retains performance and alignment as +a canonical image network becomes deeper. Neither result alone is presented +as proving the other. + +## Fixed panel and reuse boundary + +Use CIFAR-10 ResNet-20/32/56 with width 16, BatchNorm, option-A shortcuts, +batch size 128, all 50,000 training examples, 200 epochs, SGD momentum 0.9, +weight decay `1e-4`, base learning rate 0.1 with 10-fold drops at epochs 100 +and 150, no warmup, and model/data-loader seeds 10--14. Test is evaluated +exactly once at the endpoint; validation and intermediate test evaluations are +zero. + +The matched methods are: + +1. exact BP on the identical forward architecture; +2. tuned DFA at its already selected hidden rate 0.03 and output rate 0.1; +3. clean reciprocal Kolen--Pollack at hidden/output rate 0.1; and +4. dynamic innovation on the same reciprocal substrate, with traffic ratio 4, + traffic seed `5000 + model_seed`, 64 instruction-off calibration examples, + one closed-form neutral projection, and no later predictor update. + +The already frozen D4 clean-KP and dynamic records at ResNet-20/seeds 10--14 +are reused verbatim rather than rerun. Recovery v2 therefore adds exactly 50 +records: BP and DFA at depth 20, and all four methods at depths 32 and 56. +Together with D4 this forms an exact 60-cell panel. All new records must share +one clean tracked source revision; all D4 records retain their separate frozen +source revision. No completed record may be deleted, replaced, or pooled with +a pilot. + +Fixed HFA remains closed after its preregistered short gate failed. FF, PEPITA, +canonical EP, BurstCCN, and Dual Prop retain their method-native audits and are +reported as separate baselines rather than silently moved onto this matched +standard-ResNet grid. + +## Frozen mechanism and cost invariants + +Every dynamic record must retain all D4 invariants: finite 200-epoch +trajectory; mean early-third credit alignment measured only diagnostically; +final feedback/forward cosine at least 0.98 and epoch-151--200 mean at least +0.97; instruction-to-teaching RMS error at most `1e-4`; maximum post-projection +traffic ratio and absolute soma slope at most `1e-5`; zero instruction leakage +into neutral observations; exactly 64 predictor-update examples; exactly one +neutral observation per ordinary example; and zero task-loss queries. + +At every depth, dynamic and clean KP must each use at most `1.34x` the matched +BP MAC estimate. Dynamic elementwise projection work is reported separately +and must be positive. Peak allocated memory may not exceed 8 GiB on the +authorized GTX 1080 devices. Wall time is reported but is never substituted +for MACs, queries, or memory. Every record must identify an NVIDIA GTX 1080 +and visibility restricted to authorized timan107 physical GPU 5 or 7. + +## Frozen oral-A gate + +The five model/data seeds are the independent paired units. One-sided 95% +Student-t bounds use four degrees of freedom. Recovery v2 passes only if every +record and every audited value is finite and all of the following hold: + +1. Mean BP accuracy is at least 90% at every depth and every dynamic endpoint + is at least 87%. +2. At each depth, mean dynamic accuracy is within 2 points of matched BP and + the one-sided 95% upper bound on the paired deficit is at most 3 points. +3. At each depth, mean dynamic accuracy is within 1.5 points of clean KP and + the upper bound on that paired deficit is at most 2.5 points. +4. At depth 56, dynamic exceeds tuned DFA by at least 2 points on average and + the one-sided lower bound on the paired advantage is at least 1 point. +5. Dynamic gains at least 0.5 points from depth 20 to 56 on average, its + one-sided lower bound is nonnegative, and at least four of five paired + seeds improve. Its mean depth gain must also be no more than 1 point below + BP's paired depth gain. +6. Mean dynamic early-third alignment is at least 0.85 at every depth; every + depth-56 seed is at least 0.80; and the depth-56 mean retains at least 90% + of the depth-20 mean. +7. Every mechanism, query, MAC, elementwise-work, memory, source, split, and + evaluation invariant above passes without exception. + +The executable runner and immutable complete-grid analyzer are +`experiments/oral_a_dynamic_scaling_v2.py` and +`experiments/analyze_oral_a_dynamic_scaling_v2.py`. A complete pass raises the +formal milestone from 8 to 9 and establishes the standard-depth oral-A claim. +Any failure is retained, leaves the score at 8, and closes this branch without +depth, seed, optimizer, or threshold repair. diff --git a/experiments/analyze_oral_a_dynamic_scaling_v2.py b/experiments/analyze_oral_a_dynamic_scaling_v2.py new file mode 100644 index 0000000..c593929 --- /dev/null +++ b/experiments/analyze_oral_a_dynamic_scaling_v2.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +"""Audit the calibrated-BCI-v2-gated standard-depth confirmation panel.""" +import argparse +import glob +import hashlib +import json +import os +import statistics +import subprocess + +from analyze_oral_a_dynamic_scaling import ( + DEPTHS, + METHODS, + SEEDS, + dynamic_invariants, + oral_a_checks, + require, + sha256, + validate_record, +) +from oral_a_dynamic_scaling_v2 import require_prerequisites + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROTOCOL_PATH = os.path.join(ROOT, "ORAL_A_RECOVERY_V2.md") +RUNNER_PATH = os.path.join( + ROOT, "experiments", "oral_a_dynamic_scaling_v2.py") +ANALYZER_PATH = os.path.abspath(__file__) +LEGACY_RUNNER_PATH = os.path.join( + ROOT, "experiments", "oral_a_dynamic_scaling.py") +LEGACY_ANALYZER_PATH = os.path.join( + ROOT, "experiments", "analyze_oral_a_dynamic_scaling.py") +CONV_RUN_PATH = os.path.join(ROOT, "experiments", "conv_run.py") + + +def git_blob_sha256(commit, relative_path): + content = subprocess.run( + ["git", "show", f"{commit}:{relative_path}"], + cwd=ROOT, + check=True, + capture_output=True, + ).stdout + return hashlib.sha256(content).hexdigest() + + +def require_training_source_bound(commit): + paths = ( + PROTOCOL_PATH, + RUNNER_PATH, + LEGACY_RUNNER_PATH, + CONV_RUN_PATH, + ) + for path in paths: + relative = os.path.relpath(path, ROOT) + require( + git_blob_sha256(commit, relative) == sha256(path), + f"training source drift after {commit}: {relative}", + ) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--d4_results", default="results/kp_dynamic_projection_confirmation") + parser.add_argument( + "--new_results", default="results/oral_a_dynamic_scaling_v2") + parser.add_argument( + "--d4_gate", + default="results/kp_dynamic_projection_confirmation_gate.json") + parser.add_argument( + "--bci_v2_gate", + default="results/bci_v2_calibrated_confirmation_gate.json") + parser.add_argument( + "--out", default="results/oral_a_dynamic_scaling_v2_gate.json") + args = parser.parse_args() + prerequisite = require_prerequisites(args.d4_gate, args.bci_v2_gate) + with open(args.d4_gate) as handle: + d4 = json.load(handle) + with open(args.bci_v2_gate) as handle: + bci_v2 = json.load(handle) + + expected_d4 = { + f"seed{seed}_{condition}.json" + for seed in SEEDS + for condition in ("clean_kp", "dynamic") + } + observed_d4 = { + os.path.basename(path) + for path in glob.glob(os.path.join(args.d4_results, "*.json")) + } + require( + observed_d4 == expected_d4, + f"D4 record drift: missing={sorted(expected_d4-observed_d4)}, " + f"extra={sorted(observed_d4-expected_d4)}", + ) + expected_new = { + f"{method}_d{depth}_s{seed}.json" + for depth in DEPTHS + for seed in SEEDS + for method in METHODS + if not (depth == 20 and method in ("clean_kp", "dynamic")) + } + observed_new = { + os.path.basename(path) + for path in glob.glob(os.path.join(args.new_results, "*.json")) + } + require( + observed_new == expected_new, + f"new record drift: missing={sorted(expected_new-observed_new)}, " + f"extra={sorted(observed_new-expected_new)}", + ) + + records = {} + rows = {} + source_sha256 = {} + d4_commits = set() + new_commits = set() + for seed in SEEDS: + for method, condition in ( + ("clean_kp", "clean_kp"), + ("dynamic", "dynamic"), + ): + path = os.path.join( + args.d4_results, f"seed{seed}_{condition}.json") + with open(path) as handle: + record = json.load(handle) + records[(method, 20, seed)] = record + rows[(method, 20, seed)] = validate_record( + record, path, method, 20, seed) + d4_commits.add(rows[(method, 20, seed)]["source_commit"]) + source_sha256[path] = sha256(path) + for depth in DEPTHS: + for seed in SEEDS: + for method in METHODS: + if depth == 20 and method in ("clean_kp", "dynamic"): + continue + path = os.path.join( + args.new_results, f"{method}_d{depth}_s{seed}.json") + with open(path) as handle: + record = json.load(handle) + records[(method, depth, seed)] = record + rows[(method, depth, seed)] = validate_record( + record, path, method, depth, seed) + new_commits.add(rows[(method, depth, seed)]["source_commit"]) + source_sha256[path] = sha256(path) + + require(len(rows) == 60, "oral-A grid must contain exactly 60 records") + require( + len(d4_commits) == 1 and len(new_commits) == 1, + "source revision drift", + ) + require( + next(iter(d4_commits)) == d4["metrics"]["source_commit"], + "D4 source does not match its gate", + ) + new_commit = next(iter(new_commits)) + require_training_source_bound(new_commit) + require( + new_commit == prerequisite["git_commit"], + "analysis must run at the exact clean training revision", + ) + + failures = [] + mechanism = {} + for depth in DEPTHS: + for seed in SEEDS: + dynamic = records[("dynamic", depth, seed)] + mechanism[f"d{depth}_s{seed}"] = dynamic_invariants( + dynamic, depth, seed, failures) + bp_macs = rows[("bp", depth, seed)]["total_macs"] + for method in ("clean_kp", "dynamic"): + if rows[(method, depth, seed)]["total_macs"] > 1.34 * bp_macs: + failures.append(f"{method}_d{depth}_s{seed}:mac_cost") + if rows[("dynamic", depth, seed)]["peak_memory"] > 8 * 1024 ** 3: + failures.append(f"dynamic_d{depth}_s{seed}:peak_memory") + for method in ("dfa", "clean_kp", "dynamic"): + if rows[(method, depth, seed)]["logical_queries"] != 0: + failures.append(f"{method}_d{depth}_s{seed}:queries") + + accuracies = { + method: { + depth: [ + rows[(method, depth, seed)]["accuracy"] for seed in SEEDS + ] + for depth in DEPTHS + } + for method in METHODS + } + alignments = { + depth: [ + rows[("dynamic", depth, seed)]["early_alignment"] + for seed in SEEDS + ] + for depth in DEPTHS + } + checks, derived = oral_a_checks(accuracies, alignments, failures) + passed = all(checks.values()) + output = { + "protocol": "oral_a_dynamic_innovation_scaling_recovery_v2", + "status": "passed" if passed else "failed", + "complete_grid": True, + "checks": checks, + "metrics": { + "accuracy_by_method_depth_seed": accuracies, + "mean_accuracy": { + method: { + str(depth): statistics.mean(values) + for depth, values in by_depth.items() + } + for method, by_depth in accuracies.items() + }, + **derived, + "dynamic_alignment": alignments, + "mechanism": mechanism, + "invariant_failures": failures, + }, + "d4_source_commit": next(iter(d4_commits)), + "new_source_commit": new_commit, + "source_sha256": source_sha256, + "protocol_sha256": sha256(PROTOCOL_PATH), + "runner_sha256": sha256(RUNNER_PATH), + "analyzer_sha256": sha256(ANALYZER_PATH), + "legacy_runner_sha256": sha256(LEGACY_RUNNER_PATH), + "legacy_analyzer_sha256": sha256(LEGACY_ANALYZER_PATH), + "conv_run_sha256": sha256(CONV_RUN_PATH), + "d4_gate_sha256": prerequisite["d4_gate_sha256"], + "bci_v2_gate_sha256": prerequisite["bci_v2_gate_sha256"], + "bci_v2_protocol": bci_v2["protocol"], + "test_evaluations": 60, + "standard_depth_scaling_established": passed, + "old_oral_a_failures_preserved": True, + "review_score_before": 8, + "review_score_after": 9 if passed else 8, + "score_change_rule": ( + "only the complete calibrated-BCI-v2-gated 60-cell " + "standard-ResNet panel establishes oral-A scaling" + ), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + if os.path.exists(args.out): + with open(args.out) as handle: + existing = json.load(handle) + require( + existing == output, + "existing oral-A recovery-v2 gate differs from re-audit", + ) + print(json.dumps(output, indent=2)) + return + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/oral_a_dynamic_scaling_v2.py b/experiments/oral_a_dynamic_scaling_v2.py new file mode 100644 index 0000000..2e58720 --- /dev/null +++ b/experiments/oral_a_dynamic_scaling_v2.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Run shards of the calibrated-BCI-v2-gated standard-depth panel.""" +import argparse +import hashlib +import json +import os +import subprocess + +from oral_a_dynamic_scaling import ( + DEPTHS, + METHODS, + SEEDS, + existing_matches, + jobs, +) + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROTOCOL_PATH = os.path.join(ROOT, "ORAL_A_RECOVERY_V2.md") +LEGACY_RUNNER_PATH = os.path.join( + ROOT, "experiments", "oral_a_dynamic_scaling.py") + + +def sha256(path): + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_output(*args): + return subprocess.run( + ["git", *args], cwd=ROOT, check=True, capture_output=True, + text=True).stdout.strip() + + +def require_prerequisites(d4_path, bci_v2_path): + with open(d4_path) as handle: + d4 = json.load(handle) + with open(bci_v2_path) as handle: + bci_v2 = json.load(handle) + if not ( + d4.get("protocol") == "kp_dynamic_neutral_projection_confirmation_v1" + and d4.get("status") == "passed" + and d4.get("review_score_after") == 7 + ): + raise ValueError("oral-A recovery v2 requires the complete D4 pass") + if not ( + bci_v2.get("protocol") + == "oral_b_v2_calibrated_recovery_confirmation_v1" + and bci_v2.get("status") == "passed" + and bci_v2.get("complete_grid") is True + and bci_v2.get("oral_b_v2_outcome_surprise_established") is True + and bci_v2.get("review_score_after") == 8 + and bci_v2.get("new_oral_a_v2_protocol_may_be_frozen") is True + and bci_v2.get("old_oral_a_gate_remains_closed") is True + and bci_v2.get("all_prior_failures_preserved") is True + ): + raise ValueError( + "oral-A recovery v2 requires the complete calibrated BCI-v2 pass") + + paths = [ + os.path.abspath(__file__), + PROTOCOL_PATH, + LEGACY_RUNNER_PATH, + os.path.abspath(d4_path), + os.path.abspath(bci_v2_path), + os.path.join(ROOT, "experiments", "conv_run.py"), + ] + relative = [os.path.relpath(path, ROOT) for path in paths] + tracked = all( + subprocess.run( + ["git", "ls-files", "--error-unmatch", path], + cwd=ROOT, + capture_output=True, + ).returncode == 0 + for path in relative + ) + dirty = bool(git_output( + "status", "--porcelain", "--untracked-files=no")) + if dirty or not tracked: + raise RuntimeError( + "oral-A recovery v2 requires a clean tracked source") + return { + "git_commit": git_output("rev-parse", "HEAD"), + "protocol_sha256": sha256(PROTOCOL_PATH), + "d4_gate_sha256": sha256(d4_path), + "bci_v2_gate_sha256": sha256(bci_v2_path), + "legacy_runner_sha256": sha256(LEGACY_RUNNER_PATH), + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--d4_gate", + default="results/kp_dynamic_projection_confirmation_gate.json", + ) + parser.add_argument( + "--bci_v2_gate", + default="results/bci_v2_calibrated_confirmation_gate.json", + ) + parser.add_argument("--device", default="cuda") + parser.add_argument("--method", choices=("all",) + METHODS, default="all") + parser.add_argument("--depth", type=int, choices=DEPTHS) + parser.add_argument("--seed", type=int, choices=SEEDS) + parser.add_argument("--shard-index", type=int, default=0) + parser.add_argument("--num-shards", type=int, default=1) + parser.add_argument( + "--outdir", default="results/oral_a_dynamic_scaling_v2") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + if not 0 <= args.shard_index < args.num_shards: + raise ValueError("invalid shard index") + source = require_prerequisites(args.d4_gate, args.bci_v2_gate) + selected = jobs(args.device, args.outdir) + if args.method != "all": + selected = [job for job in selected if job[0] == args.method] + if args.depth is not None: + selected = [job for job in selected if job[1] == args.depth] + if args.seed is not None: + selected = [job for job in selected if job[2] == args.seed] + selected = [ + job + for index, job in enumerate(selected) + if index % args.num_shards == args.shard_index + ] + if not selected: + raise ValueError("no oral-A recovery-v2 jobs match the requested shard") + + os.makedirs(args.outdir, exist_ok=True) + for method, depth, seed, path, command in selected: + tag = f"{method}_d{depth}_s{seed}" + if os.path.exists(path): + if not existing_matches( + path, method, depth, seed, source["git_commit"] + ): + raise RuntimeError( + f"refusing mismatched existing record: {path}") + print(f"preserving {tag}", flush=True) + continue + print(tag, " ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, cwd=ROOT, check=True) + + +if __name__ == "__main__": + main() diff --git a/experiments/oral_a_dynamic_scaling_v2_smoke.py b/experiments/oral_a_dynamic_scaling_v2_smoke.py new file mode 100644 index 0000000..2fa72dd --- /dev/null +++ b/experiments/oral_a_dynamic_scaling_v2_smoke.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Endpoint-free audit of the oral-A recovery-v2 frozen funnel.""" +import copy +import json +import os +import tempfile + +from analyze_oral_a_dynamic_scaling import oral_a_checks +from oral_a_dynamic_scaling import DEPTHS, METHODS, SEEDS, jobs +from oral_a_dynamic_scaling_v2 import require_prerequisites + + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def main(): + generated = jobs("cuda", "results/oral_a_dynamic_scaling_v2") + assert len(generated) == 50 + keys = [(method, depth, seed) for method, depth, seed, _, _ in generated] + assert len(set(keys)) == 50 + expected = { + (method, depth, seed) + for depth in DEPTHS + for seed in SEEDS + for method in METHODS + if not (depth == 20 and method in ("clean_kp", "dynamic")) + } + assert set(keys) == expected + for method, depth, seed, path, command in generated: + assert path.endswith(f"{method}_d{depth}_s{seed}.json") + assert "--epochs" in command and command[command.index("--epochs")+1] == "200" + assert "--eval_every" in command + assert command[command.index("--eval_every")+1] == "0" + assert "--eval_split" in command + assert command[command.index("--eval_split")+1] == "test" + + passing = { + "bp": {20: [0.91] * 5, 32: [0.92] * 5, 56: [0.93] * 5}, + "dfa": {20: [0.80] * 5, 32: [0.81] * 5, 56: [0.84] * 5}, + "clean_kp": {20: [0.91] * 5, 32: [0.92] * 5, 56: [0.93] * 5}, + "dynamic": {20: [0.91] * 5, 32: [0.92] * 5, 56: [0.93] * 5}, + } + alignments = {20: [0.95] * 5, 32: [0.93] * 5, 56: [0.90] * 5} + checks, _ = oral_a_checks(passing, alignments, []) + assert all(checks.values()) + failing = copy.deepcopy(passing) + failing["dynamic"][56] = [0.86] * 5 + checks, _ = oral_a_checks(failing, alignments, []) + assert not all(checks.values()) + + d4_path = os.path.join( + ROOT, "results", "kp_dynamic_projection_confirmation_gate.json") + bci_path = os.path.join( + ROOT, "results", "bci_v2_calibrated_confirmation_gate.json") + with open(bci_path) as handle: + old_gate = json.load(handle) + old_gate["protocol"] = "oral_b_td_confirmation_v1" + with tempfile.TemporaryDirectory() as tmp: + fake = os.path.join(tmp, "old_r2.json") + with open(fake, "w") as handle: + json.dump(old_gate, handle) + try: + require_prerequisites(d4_path, fake) + except ValueError: + pass + else: + raise AssertionError("legacy oral-B gate must not open recovery v2") + + print("oral-A recovery-v2 endpoint-free smoke passed") + + +if __name__ == "__main__": + main() |
