diff options
| -rw-r--r-- | DYNAMIC_INNOVATION.md | 65 | ||||
| -rwxr-xr-x | experiments/analyze_kp_dynamic_projection_short.py | 223 | ||||
| -rwxr-xr-x | experiments/kp_dynamic_projection_short.py | 53 |
3 files changed, 341 insertions, 0 deletions
diff --git a/DYNAMIC_INNOVATION.md b/DYNAMIC_INNOVATION.md index 19e39da..0b59622 100644 --- a/DYNAMIC_INNOVATION.md +++ b/DYNAMIC_INNOVATION.md @@ -88,3 +88,68 @@ threshold or adding a gain/clipping hyperparameter. A pass supplies no task accuracy claim and cannot change the reviewer score. It only authorizes a new, separately committed validation protocol with an explicit cost boundary. +## Audited D1 outcome + +D1 passes all frozen checks at clean revision `8b28522`. All 352 updates and +states remain finite; maximum minibatch loss is `2.9281` and the final-32 mean +is `1.6293`. The uncontrolled neutral residual reaches `0.005833` of traffic +RMS and the fast correction slope reaches `0.02968`, so the controller is not +a numerical no-op. Projection reduces the worst neutral/traffic RMS ratio to +`3.03e-8` and the worst absolute residual-soma slope to `1.52e-8`. Maximum +forward/feedback weights are `1.181/0.833` and maximum momenta are +`0.600/0.364`. No held-out example was evaluated and the score remains 5/10. + +## D2: frozen short validation endpoint + +D1 opens exactly one 20-epoch seed-0 validation run. Use the same ResNet-20, +45k/5k CIFAR-10 split, augmentation, batch size 128, constant learning rate +0.1 (the inherited KP short schedule), momentum 0.9, decay `1e-4`, four-to-one +traffic, reciprocal KP path, zero-margin 64-example closed-form slow fit, and +fast neutral projection on every task minibatch. Evaluate the validation set +once after training, probe alignment on 32 training examples only after the +endpoint is fixed, and never evaluate test. + +D2 passes only if: + +- the final record, all 20 epoch records, feedback trajectories, signal + trajectories, projection certificates, and diagnostics are finite; +- validation accuracy is at least 80%, within three points of the frozen clean + KP short endpoint (`82.66%`), at least 70 points above each failed MT-1 raw + and norm-matched endpoint, and final loss is finite; +- early-third used-signal alignment is at least 0.70, final feedback/forward + cosine is at least 0.80, and the epoch-11--20 mean feedback cosine is at + least 0.70; +- every epoch keeps used/instruction RMS within `1e-4` of one, maximum + post-projection neutral/traffic RMS at most `1e-5`, maximum absolute + post-projection soma slope at most `1e-5`, and zero task-instruction + observations in the fast fit; +- exactly one closed-form 64-example slow fit occurs, task predictor cadence is + zero, neutral-projection observation count equals ordinary training examples, + initial traffic ratio error is at most `1e-5`, and task-loss queries are zero; +- total MACs are at most `1.34x` the frozen 20-epoch BP record, all additional + elementwise work is reported, and peak allocated memory is at most 2.5 GiB; + and +- there is exactly one validation evaluation and zero test evaluations. + +D2 failure closes the branch. A pass remains a one-seed short validation +result and leaves the paper score at 5/10, but opens the already specified D3 +full validation endpoint below. + +## D3: conditionally frozen full validation endpoint + +If and only if D2 passes, run one seed-0 200-epoch validation record with the +same method and data boundary. The only schedule change is the inherited KP-2 +step schedule: LR 0.1, drops by 0.1 at epochs 100 and 150, no warmup. Evaluate +validation exactly once at the end and test never. D3 passes only if every +D2 mechanics/cost/query certificate remains satisfied (using the corresponding +200-epoch BP cost), validation accuracy is at least 89%, lies within 1.5 points +of BP (`91.62%`) and clean KP (`91.26%`), final early alignment is at least +0.90, final feedback cosine is at least 0.98, the final-50-epoch mean feedback +cosine is at least 0.97, and all trajectories remain finite. + +A D3 pass would establish a frozen full standard-ResNet endpoint and make a +5-to-6 reviewer-score update eligible, subject to an adversarial audit. It +would then open a separately frozen independent multi-seed test confirmation; +no confirmation seed or test endpoint may be touched before that protocol is +committed. A D3 failure leaves the score at 5 and closes this branch. + diff --git a/experiments/analyze_kp_dynamic_projection_short.py b/experiments/analyze_kp_dynamic_projection_short.py new file mode 100755 index 0000000..8f33874 --- /dev/null +++ b/experiments/analyze_kp_dynamic_projection_short.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Audit the frozen D2 short dynamic-projection validation endpoint.""" +import argparse +import json +import math +import os + + +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" +BP_EPOCH20_MACS = 109_487_808_000_000 +KP_SHORT_ACCURACY = 0.8266 +MT1_ACCURACY = {"raw": 0.1, "matched": 0.1} + + +def numeric_leaves(value): + if isinstance(value, bool) or value is None: + return + if isinstance(value, (int, float)): + yield float(value) + elif isinstance(value, dict): + for child in value.values(): + yield from numeric_leaves(child) + elif isinstance(value, (list, tuple)): + for child in value: + yield from numeric_leaves(child) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--input", default="results/kp_dynamic_projection_short/dynamic.json") + parser.add_argument( + "--d1_gate", default="results/kp_dynamic_projection_gate.json") + parser.add_argument( + "--mt1_gate", default="results/kp_innovation_short_gate.json") + parser.add_argument( + "--out", default="results/kp_dynamic_projection_short_gate.json") + args = parser.parse_args() + with open(args.d1_gate) as handle: + d1 = json.load(handle) + if (d1.get("protocol") != + "kp_dynamic_neutral_projection_training_prefix_v1" + or d1.get("status") != "passed"): + raise ValueError("D2 requires the audited D1 pass") + with open(args.mt1_gate) as handle: + mt1 = json.load(handle) + if (mt1.get("protocol") != "kp_mixed_traffic_short_v1" + or mt1.get("status") != "failed"): + raise ValueError("D2 requires the frozen failed MT-1 comparator") + for rule, accuracy in MT1_ACCURACY.items(): + if float(mt1["metrics"]["accuracy"][rule]) != accuracy: + raise ValueError(f"MT-1 {rule} endpoint drift") + + with open(args.input) as handle: + record = json.load(handle) + expected = { + "mode": "kp_traffic", "traffic_rule": "innovation", + "predictor_mode": "closed_form", "neutral_projection": 1, + "depth": 20, "width": 16, "seed": 0, "loader_seed": 0, + "batch_size": 128, "epochs": 20, "train_limit": 0, + "val_examples": 5000, "split_seed": 2027, + "eval_split": "validation", "eval_every": 0, + "augment_train": 1, "lr": 0.1, "output_lr": 0.1, + "lr_schedule": "step", "lr_milestones": "100,150", + "lr_gamma": 0.1, "warmup_epochs": 0, "momentum": 0.9, + "weight_decay": 1e-4, "normalization": "batchnorm", + "a_scale": 1.0, "traffic_seed": 4000, "traffic_ratio": 4.0, + "traffic_calibration_examples": 64, "learn_P": 1, + "eta_P": 0.1, "predictor_warmup_steps": 1, + "predictor_every": 0, "alignment_probe": 32, + } + for key, value in expected.items(): + if record["args"].get(key) != value: + raise ValueError(f"D2 {key} drift") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError("tracked-dirty D2 record") + if record["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError("D2 split drift") + evaluation = record["evaluation_protocol"] + if (evaluation["validation_evaluations"] != 1 + or evaluation["test_evaluations"] != 0 + or evaluation["test_used_for_selection"] is not False): + raise ValueError("D2 evaluation boundary drift") + if record.get("calibration_metric_space") != ( + "reciprocal_local_activity_products_with_mixed_apical_traffic"): + raise ValueError("D2 metric-space drift") + warmup = record.get("predictor_warmup", {}) + if (warmup.get("mode") != "closed_form" + or warmup.get("steps") != 1 + or warmup.get("examples") != 64 + or warmup.get("instruction_present") is not False + or warmup.get("task_loader_state_restored") is not True + or warmup.get("reuses_traffic_calibration_forward") is not True): + raise ValueError("D2 neutral slow-fit invariant failed") + epochs = record["epochs"] + if len(epochs) != 20 or any(row["epoch"] != index + 1 + for index, row in enumerate(epochs)): + raise ValueError("D2 epoch trajectory is incomplete") + + projection = [row.get("neutral_projection") for row in epochs] + mixed = [row.get("mixed_apical") for row in epochs] + tracking = [row.get("feedback_tracking") for row in epochs] + if any(value is None for value in projection + mixed + tracking): + raise ValueError("D2 audited trajectory is incomplete") + final = record["final"] + diagnostics = record["diagnostics"] + accuracy = float(final["accuracy"]) + early = float(diagnostics["early_third_mean"]) + final_feedback = float(diagnostics["mean_feedback_forward_cosine"]) + late_feedback = sum(float(value["mean_feedback_forward_cosine"]) + for value in tracking[10:]) / 10 + maximum_signal_ratio_error = max(abs( + float(values["teaching_rms"]) + / max(float(values["instruction_rms"]), 1e-30) - 1.0) + for values in mixed) + maximum_post_ratio = max(float(value[ + "maximum_post_projection_traffic_rms_ratio"]) + for value in projection) + maximum_post_slope = max(float(value[ + "maximum_absolute_post_projection_soma_slope"]) + for value in projection) + maximum_pre_ratio = max(float(value[ + "maximum_pre_projection_traffic_rms_ratio"]) + for value in projection) + instruction_observations = sum(int(value["instruction_observations"]) + for value in projection) + initial_ratio_error = max(abs(float(value) - 4.0) for value in + record["traffic_calibration"][ + "realized_traffic_instruction_rms_ratio"]) + work = record["work"] + counters = record["counters"] + mac_ratio = float(work["total_macs_estimate"]) / BP_EPOCH20_MACS + all_finite = bool(final["finite"]) and all( + math.isfinite(value) for value in numeric_leaves({ + "final": final, "epochs": epochs, "diagnostics": diagnostics, + "warmup": warmup, "traffic": record["traffic_calibration"], + "work": work, + })) + + checks = { + "record_trajectory_and_diagnostics_finite": all_finite, + "accuracy_at_least_0p80": accuracy >= 0.80, + "within_3_points_of_clean_kp_short": ( + accuracy >= KP_SHORT_ACCURACY - 0.03), + "at_least_70_points_above_mt1_raw_and_matched": all( + accuracy - value >= 0.70 for value in MT1_ACCURACY.values()), + "early_alignment_at_least_0p70": early >= 0.70, + "final_feedback_cosine_at_least_0p80": final_feedback >= 0.80, + "epoch11_to20_feedback_cosine_at_least_0p70": late_feedback >= 0.70, + "used_instruction_rms_ratio_within_1e_minus_4": ( + maximum_signal_ratio_error <= 1e-4), + "post_projection_traffic_ratio_at_most_1e_minus_5": ( + maximum_post_ratio <= 1e-5), + "post_projection_soma_slope_at_most_1e_minus_5": ( + maximum_post_slope <= 1e-5), + "zero_instruction_observations_in_fast_fit": ( + instruction_observations == 0), + "one_frozen_64_example_slow_fit": ( + counters["predictor_update_examples"] == 64 + and counters["predictor_warmup_examples"] == 0 + and record["args"]["predictor_every"] == 0), + "projection_observes_each_ordinary_example": ( + counters["neutral_projection_examples"] + == counters["ordinary_examples"] == 900000), + "initial_traffic_ratio_error_at_most_1e_minus_5": ( + initial_ratio_error <= 1e-5), + "zero_task_loss_queries": work["logical_batch_loss_queries"] == 0, + "macs_at_most_1p34x_bp": mac_ratio <= 1.34, + "elementwise_and_neutral_cost_reported": ( + work["elementwise_operations_estimate"] > 0 + and work["neutral_projection_observations"] == 900000), + "peak_allocated_memory_at_most_2p5_gib": ( + record["hardware"]["peak_memory_allocated_bytes"] + <= int(2.5 * 1024 ** 3)), + "one_validation_and_zero_test_evaluations": ( + evaluation["validation_evaluations"] == 1 + and evaluation["test_evaluations"] == 0), + } + passed = all(checks.values()) + output = { + "protocol": "kp_dynamic_neutral_projection_short_v1", + "status": "passed" if passed else "failed", + "checks": checks, + "metrics": { + "accuracy": accuracy, + "loss": float(final["loss"]), + "clean_kp_short_accuracy": KP_SHORT_ACCURACY, + "gain_over_mt1": {key: accuracy - value + for key, value in MT1_ACCURACY.items()}, + "early_third_alignment": early, + "final_feedback_forward_cosine": final_feedback, + "epoch11_to20_feedback_forward_cosine": late_feedback, + "maximum_used_instruction_rms_ratio_error": ( + maximum_signal_ratio_error), + "maximum_pre_projection_traffic_rms_ratio": maximum_pre_ratio, + "maximum_post_projection_traffic_rms_ratio": maximum_post_ratio, + "maximum_post_projection_soma_slope": maximum_post_slope, + "total_macs": int(work["total_macs_estimate"]), + "mac_ratio_to_bp": mac_ratio, + "elementwise_operations_estimate": int( + work["elementwise_operations_estimate"]), + "peak_memory_allocated_bytes": int(record["hardware"][ + "peak_memory_allocated_bytes"]), + "wall_s": float(record["timing"]["total_timed_wall_s"]), + "source_commit": record["provenance"]["git_commit"], + }, + "full_validation_opened": passed, + "confirmation_test_seeds_touched": False, + "review_score_before": 5, + "review_score_after": 5, + "score_change_rule": ( + "a one-seed short validation endpoint cannot raise the score"), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() + diff --git a/experiments/kp_dynamic_projection_short.py b/experiments/kp_dynamic_projection_short.py new file mode 100755 index 0000000..79da620 --- /dev/null +++ b/experiments/kp_dynamic_projection_short.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Run the frozen D2 short dynamic-projection validation endpoint.""" +import argparse +import json +import os +import subprocess +import sys + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--device", default="cuda") + parser.add_argument("--dry_run", action="store_true") + parser.add_argument( + "--d1_gate", default="results/kp_dynamic_projection_gate.json") + parser.add_argument( + "--out", default="results/kp_dynamic_projection_short/dynamic.json") + args = parser.parse_args() + with open(args.d1_gate) as handle: + gate = json.load(handle) + if (gate.get("protocol") != + "kp_dynamic_neutral_projection_training_prefix_v1" + or gate.get("status") != "passed"): + raise ValueError("D2 requires the audited D1 pass") + command = [ + sys.executable, "experiments/conv_run.py", + "--mode", "kp_traffic", "--traffic_rule", "innovation", + "--predictor_mode", "closed_form", "--neutral_projection", "1", + "--device", args.device, "--depth", "20", "--width", "16", + "--seed", "0", "--loader_seed", "0", "--batch_size", "128", + "--epochs", "20", "--train_limit", "0", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "0", + "--augment_train", "1", "--lr", "0.1", "--output_lr", "0.1", + "--lr_schedule", "step", "--lr_milestones", "100,150", + "--lr_gamma", "0.1", "--warmup_epochs", "0", + "--momentum", "0.9", "--weight_decay", "1e-4", + "--normalization", "batchnorm", "--a_scale", "1", + "--traffic_seed", "4000", "--traffic_ratio", "4", + "--traffic_calibration_examples", "64", + "--learn_P", "1", "--eta_P", "0.1", + "--predictor_warmup_steps", "1", "--predictor_every", "0", + "--alignment_probe", "32", "--out", args.out, + ] + print(" ".join(command), flush=True) + if not args.dry_run: + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() + |