diff options
| -rw-r--r-- | MIXED_TRAFFIC.md | 145 | ||||
| -rw-r--r-- | experiments/analyze_kp_innovation_short.py | 182 | ||||
| -rw-r--r-- | experiments/kp_innovation_short_development.py | 56 |
3 files changed, 383 insertions, 0 deletions
diff --git a/MIXED_TRAFFIC.md b/MIXED_TRAFFIC.md new file mode 100644 index 0000000..0bc0ee2 --- /dev/null +++ b/MIXED_TRAFFIC.md @@ -0,0 +1,145 @@ +# KP mixed-traffic innovation protocol + +## Question and attribution boundary + +This protocol asks whether the Harnett-specific somato-dendritic innovation is +load-bearing on a standard ResNet once a strong non-backpropagating credit path +is available. The reciprocal Kolen--Pollack (KP) pathway is inherited from +Akrout et al. and receives no SDIL novelty credit. The candidate contribution +is only the neutral-period subtraction used in the mixed apical compartment. + +At every hidden population, first compute the hierarchical KP instruction +`s_l`. Add fixed soma-predictable traffic + +```text +t_l = g_l B_l h_l, +a_l = s_l + t_l, +a_hat_l = P_l h_l + b_l, +r_l = a_l - a_hat_l. +``` + +`B_l` is a per-unit log-normal coefficient drawn once from traffic seed 4000. +The positive gain `g_l` is fixed once at initialization using the first 64 +unaugmented development-training examples so that every layer has +`RMS(t_l)/RMS(s_l)=4`. Labels enter only this one training-prefix scale +calibration through `s_l`; no validation endpoint is used. The target ratio is +not retuned across signals, epochs, or seeds. + +The diagonal affine predictor is trained only with instruction switched off. +All three conditions receive exactly 20 neutral warmup minibatches at +`eta_P=0.1` and one neutral predictor update every 16 task minibatches. Raw and +norm-matched controls pay the same predictor-learning cost even though raw +does not consume its prediction. The three used signals are + +```text +raw: u_l = a_l, +matched: u_l = ||r_l||_example / ||a_l||_example * a_l, +innovation: u_l = r_l. +``` + +Norms are taken over every non-batch coordinate separately for each example +and layer. Consequently the matched control has the innovation's update-signal +norm but the raw signal's direction at the same network state. Every condition +uses its own used signal for both the forward local correlation and the +independently recomputed reciprocal KP correlation; matched optimizer and +decay dynamics are otherwise unchanged. + +This is a controlled predictable-traffic test, not the Oral-B natural-traffic +claim. The broad endogenous top-down traffic and desired-velocity branches +remain failed and are not reopened by this protocol. + +## MT-0: mechanics and cost gate + +Before any task endpoint, the float64/CPU smoke suite must establish all of: + +- setting traffic gain to zero makes raw, matched, and innovation exactly the + clean KP instruction when `P=b=0`; +- setting `P_l=g_l B_l, b_l=0` removes the predictable traffic exactly; +- layerwise gain calibration realizes ratio 4 to relative error below `1e-12`; +- matched raw has the innovation norm to relative error below `1e-12` and + preserves raw cosine direction to absolute error below `1e-12`; +- the predictor update depends only on local soma and instruction-off apical + traffic, not labels, output errors, or forward/feedback weights; +- independently recomputed forward and reciprocal correlations remain equal + under every used signal, and no parameter acquires an autograd graph; +- affine MACs, predictor/traffic elementwise operations, loss queries, memory, + and wall time are reported separately. All three conditions execute the + same calibration and predictor schedule. + +Failure closes this branch. Passing mechanics does not change the reviewer +score. + +## MT-1: frozen 20-epoch load-bearing screen + +This screen is conditional on a KP-2 pass. Run exactly raw, matched, and +innovation at seed 0 on the frozen 45,000/5,000 CIFAR-10 split. Copy KP-1: +ResNet-20 width 16, batch 128, standard augmentation, BatchNorm, hidden/output +LR 0.1, epoch-100/150 step drops (hence constant in this screen), no optimizer +warmup, momentum 0.9, and weight decay `1e-4`. Use the traffic and predictor +settings above and a 32-example final alignment audit. There is no traffic, +predictor, learning-rate, decay, or signal-rule grid. + +MT-1 passes only if all of the following hold: + +- all three records, epoch losses, tracking trajectories, and diagnostics are + finite, clean, and validation-only; +- innovation accuracy is at least 75% and no more than 5 points below the + frozen clean KP-1 accuracy of 82.66%; +- innovation exceeds raw by at least 5 accuracy points and norm-matched raw by + at least 3 points; +- innovation's used-signal early-third alignment is at least 0.70, and on the + same final innovation network its innovation alignment exceeds raw-apical + alignment by at least 0.15; +- innovation's final feedback/forward cosine is at least 0.80 and its + epoch-11--20 mean is at least 0.70; +- every initial layer ratio is within `1e-5` of 4, the post-warmup predictor + traffic-residual RMS ratio is at most 0.25, and matched norm relative error + is at most `1e-6`; +- feedback learning uses zero task-loss queries and each condition's affine + MAC estimate is at most `1.40x` the frozen epoch-20 BP reference. + +The accuracy gaps and same-state alignment rotation are conjunctive: clipping +an unstable raw update is not enough. MT-1 failure closes the branch without a +weaker traffic ratio. A pass opens MT-2 but leaves the reviewer score at 5/10. + +## MT-2: frozen full validation gate + +Copy MT-1 exactly for 200 epochs. The three jobs remain seed 0 and do not touch +the test set. In addition to finite/clean provenance, exact ratio calibration, +zero queries, matched norms, and the `1.40x` BP MAC ceiling, require: + +- innovation validation accuracy at least 88% and within 3 points of KP-2; +- innovation gains of at least 5 points over raw and 3 over matched raw; +- innovation used-signal early alignment at least 0.80 and a same-network + advantage of at least 0.15 over raw-apical alignment; +- final and epoch-151--200 mean feedback/forward cosine both at least 0.95; +- post-training traffic-residual RMS ratio at most 0.05. + +MT-2 failure closes the confirmation panel. A pass is strong development +evidence but one seed is insufficient for acceptance, so the formal score +remains 5/10. + +## MT-3: untouched five-seed test confirmation + +Only an MT-2 pass opens seeds 10--14. For each seed, pair an independently +drawn traffic field with seed `5000 + model_seed`. Freeze the chosen method and +run clean KP, raw, matched, and innovation after refitting on all 50,000 +CIFAR-10 training examples. Evaluate the test set exactly once per run; no +validation endpoint, seed deletion, or replacement is permitted. + +The confirmation passes only if all 20 records are finite and clean, mean +innovation test accuracy is at least 88%, and: + +- the mean paired innovation deficit to clean KP is at most 3 points; +- mean paired innovation gains are at least 5 points over raw and 3 over + matched raw; +- the one-sided 95% lower confidence bound of each paired innovation gain is + above zero; +- at least four of five paired gains are positive against each control; +- all mechanism, query, and cost invariants from MT-2 remain satisfied. + +A complete pass raises the strict reviewer estimate from 5/10 to 6/10 (weak +accept): it would establish a Harnett-specific, norm-controlled innovation +effect on a standard ResNet across untouched network and traffic draws. It +would not establish natural biological traffic or broad architecture scaling; +those remain Oral-B and Oral-A work respectively. diff --git a/experiments/analyze_kp_innovation_short.py b/experiments/analyze_kp_innovation_short.py new file mode 100644 index 0000000..4c70575 --- /dev/null +++ b/experiments/analyze_kp_innovation_short.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Audit and gate the frozen MT-1 mixed-traffic innovation panel.""" +import argparse +import json +import math +import os + + +RULES = ("raw", "matched", "innovation") +SPLIT_HASH = "8328b206a97c420e49e54e3eca4abe3274c4756b084355784ea3fb8059e4515b" +BP_EPOCH20_MACS = 109_487_808_000_000 +KP_SHORT_ACCURACY = 0.8266 + + +def mean_early(values): + count = max(1, len(values) // 3) + return sum(float(value) for value in values[:count]) / count + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--input_dir", default="results/kp_innovation_short") + parser.add_argument("--kp_full_gate", default="results/kp_full_gate.json") + parser.add_argument("--out", default="results/kp_innovation_short_gate.json") + args = parser.parse_args() + with open(args.kp_full_gate) as handle: + kp_gate = json.load(handle) + if (kp_gate.get("protocol") != "kolen_pollack_full_v1" + or kp_gate.get("status") != "passed"): + raise ValueError("a KP-2 pass is required") + + records = {} + expected_common = { + "mode": "kp_traffic", "depth": 20, "width": 16, "seed": 0, + "loader_seed": 0, "batch_size": 128, "epochs": 20, + "train_limit": 0, "val_examples": 5000, "split_seed": 2027, + "eval_split": "validation", "eval_every": 0, + "augment_train": 1, "lr": 0.1, "output_lr": 0.1, + "lr_schedule": "step", "lr_milestones": "100,150", + "lr_gamma": 0.1, "warmup_epochs": 0, "momentum": 0.9, + "weight_decay": 1e-4, "normalization": "batchnorm", + "a_scale": 1.0, "traffic_seed": 4000, "traffic_ratio": 4.0, + "traffic_calibration_examples": 64, "learn_P": 1, "eta_P": 0.1, + "predictor_warmup_steps": 20, "predictor_every": 16, + "alignment_probe": 32, + } + source_commits = set() + for rule in RULES: + with open(os.path.join(args.input_dir, f"{rule}.json")) as handle: + record = json.load(handle) + records[rule] = record + run_args = record["args"] + for key, value in {**expected_common, "traffic_rule": rule}.items(): + if run_args.get(key) != value: + raise ValueError(f"MT-1 {rule} {key} drift") + if record["provenance"]["git_tracked_dirty"]: + raise ValueError(f"tracked-dirty MT-1 {rule} result") + source_commits.add(record["provenance"]["git_commit"]) + if record["split"]["validation_index_sha256"] != SPLIT_HASH: + raise ValueError(f"MT-1 {rule} split drift") + protocol = record["evaluation_protocol"] + if protocol["test_evaluations"] or protocol["test_used_for_selection"]: + raise ValueError(f"MT-1 {rule} touched test") + if record.get("calibration_metric_space") != ( + "reciprocal_local_activity_products_with_mixed_apical_traffic"): + raise ValueError(f"MT-1 {rule} metric-space drift") + if len(source_commits) != 1: + raise ValueError("MT-1 conditions must share one source revision") + + accuracies = {rule: float(records[rule]["final"]["accuracy"]) + for rule in RULES} + diagnostics = {rule: records[rule]["diagnostics"] for rule in RULES} + innovation_early = float(diagnostics["innovation"]["early_third_mean"]) + innovation_raw_early = mean_early( + diagnostics["innovation"]["raw_negative_gradient_cosine"]) + tracking = { + rule: [row.get("feedback_tracking") for row in records[rule]["epochs"]] + for rule in RULES + } + trajectory_values = [] + for rule in RULES: + if len(tracking[rule]) != 20 or any(value is None for value in tracking[rule]): + raise ValueError(f"MT-1 {rule} tracking trajectory is incomplete") + for row, values in zip(records[rule]["epochs"], tracking[rule]): + trajectory_values.extend([ + float(row["train_loss"]), + float(values["mean_feedback_forward_cosine"]), + float(values["mean_feedback_forward_relative_error"]), + float(values["min_feedback_forward_cosine"]), + float(values["max_feedback_forward_relative_error"]), + ]) + + all_finite = all(record["final"]["finite"] for record in records.values()) + all_finite = all_finite and all(math.isfinite(value) for value in ( + trajectory_values + list(accuracies.values()) + [innovation_early, + innovation_raw_early])) + initial_ratio_errors = [] + predictor_residual_ratios = [] + total_macs = {} + queries = {} + elementwise = {} + for rule, record in records.items(): + initial_ratio_errors.extend(abs(float(value) - 4.0) for value in + record["traffic_calibration"]["realized_traffic_instruction_rms_ratio"]) + predictor_residual_ratios.append(float( + record["predictor_warmup"]["post_warmup_traffic_residual_rms_ratio"])) + total_macs[rule] = int(record["work"]["total_macs_estimate"]) + queries[rule] = int(record["work"]["logical_batch_loss_queries"]) + elementwise[rule] = int(record["work"]["elementwise_operations_estimate"]) + matched_norm_error = float( + diagnostics["matched"]["max_norm_match_relative_error"]) + final_feedback_cosine = float( + diagnostics["innovation"]["mean_feedback_forward_cosine"]) + late_feedback_cosine = sum(float(value["mean_feedback_forward_cosine"]) + for value in tracking["innovation"][10:]) / 10 + + checks = { + "records_trajectories_and_diagnostics_finite": all_finite, + "innovation_accuracy_at_least_0.75": accuracies["innovation"] >= 0.75, + "innovation_within_5_points_of_clean_kp": ( + accuracies["innovation"] >= KP_SHORT_ACCURACY - 0.05), + "innovation_gain_over_raw_at_least_0.05": ( + accuracies["innovation"] - accuracies["raw"] >= 0.05), + "innovation_gain_over_matched_at_least_0.03": ( + accuracies["innovation"] - accuracies["matched"] >= 0.03), + "innovation_early_alignment_at_least_0.70": innovation_early >= 0.70, + "same_network_alignment_gain_over_raw_at_least_0.15": ( + innovation_early - innovation_raw_early >= 0.15), + "final_feedback_cosine_at_least_0.80": final_feedback_cosine >= 0.80, + "epoch11_to20_feedback_cosine_at_least_0.70": ( + late_feedback_cosine >= 0.70), + "initial_layer_ratio_error_at_most_1e-5": ( + max(initial_ratio_errors) <= 1e-5), + "post_warmup_predictor_residual_ratio_at_most_0.25": ( + max(predictor_residual_ratios) <= 0.25), + "matched_norm_relative_error_at_most_1e-6": matched_norm_error <= 1e-6, + "zero_task_loss_queries": all(value == 0 for value in queries.values()), + "each_affine_mac_count_at_most_1.40x_bp": all( + value <= 1.40 * BP_EPOCH20_MACS for value in total_macs.values()), + "elementwise_cost_reported": all(value > 0 for value in elementwise.values()), + } + status = "passed" if all(checks.values()) else "failed" + output = { + "protocol": "kp_mixed_traffic_short_v1", "status": status, + "checks": checks, + "metrics": { + "accuracy": accuracies, + "clean_kp_accuracy": KP_SHORT_ACCURACY, + "innovation_gain_over_raw": ( + accuracies["innovation"] - accuracies["raw"]), + "innovation_gain_over_matched": ( + accuracies["innovation"] - accuracies["matched"]), + "innovation_early_third_alignment": innovation_early, + "same_innovation_network_raw_early_third_alignment": innovation_raw_early, + "final_feedback_forward_cosine": final_feedback_cosine, + "epoch11_to20_feedback_forward_cosine": late_feedback_cosine, + "max_initial_layer_ratio_error": max(initial_ratio_errors), + "max_post_warmup_predictor_residual_rms_ratio": ( + max(predictor_residual_ratios)), + "matched_norm_relative_error": matched_norm_error, + "total_macs": total_macs, + "mac_ratio_to_bp": {rule: value / BP_EPOCH20_MACS + for rule, value in total_macs.items()}, + "elementwise_operations_estimate": elementwise, + "logical_batch_loss_queries": queries, + "source_commit": next(iter(source_commits)), + }, + "full_validation_opened": status == "passed", + "confirmation_test_seeds_touched": False, + "review_score_before": 5, "review_score_after": 5, + "score_change_rule": ( + "a one-seed short mechanism screen cannot raise the review score"), + } + os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True) + with open(args.out, "w") as handle: + json.dump(output, handle, indent=2, sort_keys=True) + handle.write("\n") + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/experiments/kp_innovation_short_development.py b/experiments/kp_innovation_short_development.py new file mode 100644 index 0000000..a4c5645 --- /dev/null +++ b/experiments/kp_innovation_short_development.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Run the three frozen MT-1 mixed-traffic development records.""" +import argparse +import json +import os +import subprocess +import sys + + +RULES = ("raw", "matched", "innovation") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--kp_full_gate", default="results/kp_full_gate.json") + parser.add_argument("--rule", choices=("all",) + RULES, default="all") + parser.add_argument("--device", default="cuda") + parser.add_argument("--dry_run", action="store_true") + args = parser.parse_args() + with open(args.kp_full_gate) as handle: + kp_gate = json.load(handle) + if kp_gate.get("protocol") != "kolen_pollack_full_v1": + raise ValueError("unexpected KP-2 protocol") + if kp_gate.get("status") != "passed": + raise ValueError("KP-2 did not open MT-1") + + rules = RULES if args.rule == "all" else (args.rule,) + os.makedirs("results/kp_innovation_short", exist_ok=True) + for rule in rules: + command = [ + sys.executable, "experiments/conv_run.py", + "--mode", "kp_traffic", "--traffic_rule", rule, + "--device", args.device, "--depth", "20", "--width", "16", + "--seed", "0", "--loader_seed", "0", "--batch_size", "128", + "--epochs", "20", "--train_limit", "0", + "--val_examples", "5000", "--split_seed", "2027", + "--eval_split", "validation", "--eval_every", "0", + "--augment_train", "1", "--lr", "0.1", "--output_lr", "0.1", + "--lr_schedule", "step", "--lr_milestones", "100,150", + "--lr_gamma", "0.1", "--warmup_epochs", "0", + "--momentum", "0.9", "--weight_decay", "1e-4", + "--normalization", "batchnorm", "--a_scale", "1", + "--traffic_seed", "4000", "--traffic_ratio", "4", + "--traffic_calibration_examples", "64", + "--learn_P", "1", "--eta_P", "0.1", + "--predictor_warmup_steps", "20", "--predictor_every", "16", + "--alignment_probe", "32", + "--out", f"results/kp_innovation_short/{rule}.json", + ] + print(" ".join(command), flush=True) + if not args.dry_run: + subprocess.run(command, check=True) + + +if __name__ == "__main__": + main() |