paper: add evidence-bound ICLR manuscript draft

3 files changed, 301 insertions, 2 deletions

diff --git a/experiments/audit_manuscript.py b/experiments/audit_manuscript.py
new file mode 100644
index 0000000..5743f24
--- /dev/null
+++ b/experiments/audit_manuscript.py
@@ -0,0 +1,257 @@
+#!/usr/bin/env python3
+"""Bind publication-facing manuscript claims to audited result sources."""
+import argparse
+import hashlib
+import json
+import os
+import re
+
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+EXPECTED_SECTIONS = [
+    "## Abstract",
+    "## 1. Introduction",
+    "## 2. Somato-dendritic innovation learning",
+    "## 3. What residualization guarantees—and what it does not",
+    "## 4. Experimental protocol",
+    "## 5. Results",
+    "## 6. Biological-signature test and negative evidence",
+    "## 7. Related work",
+    "## 8. Limitations and discussion",
+    "## 9. Reproducibility statement",
+]
+
+
+def absolute(path):
+    return path if os.path.isabs(path) else os.path.join(ROOT, path)
+
+
+def load_json(path):
+    with open(path) as handle:
+        return json.load(handle)
+
+
+def sha256(path):
+    digest = hashlib.sha256()
+    with open(path, "rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def require(condition, message):
+    if not condition:
+        raise RuntimeError(message)
+
+
+def pointer_get(value, pointer):
+    require(pointer.startswith("/"), f"invalid JSON pointer: {pointer}")
+    current = value
+    for part in pointer[1:].split("/"):
+        part = part.replace("~1", "/").replace("~0", "~")
+        if isinstance(current, list):
+            current = current[int(part)]
+        else:
+            require(part in current, f"missing JSON pointer component {part!r}")
+            current = current[part]
+    return current
+
+
+def render(value, format_name):
+    require(
+        isinstance(value, (int, float)) and not isinstance(value, bool),
+        f"claim source is not numeric: {value!r}",
+    )
+    value = float(value)
+    if format_name == "fixed0_comma":
+        return f"{value:,.0f}"
+    if format_name == "fixed1":
+        return f"{value:.1f}"
+    if format_name == "fixed2":
+        return f"{value:.2f}"
+    if format_name == "fixed2_percent":
+        return f"{value:.2f}%"
+    if format_name == "fixed3":
+        return f"{value:.3f}"
+    if format_name == "fixed3_percent":
+        return f"{value:.3f}%"
+    if format_name == "fixed4":
+        return f"{value:.4f}"
+    if format_name == "fixed6":
+        return f"{value:.6f}"
+    if format_name == "percent3":
+        return f"{100 * value:.3f}"
+    if format_name == "percent2_percent":
+        return f"{100 * value:.2f}%"
+    if format_name == "percent3_percent":
+        return f"{100 * value:.3f}%"
+    if format_name == "abs_percent3":
+        return f"{100 * abs(value):.3f}"
+    raise RuntimeError(f"unknown claim format: {format_name}")
+
+
+def false_check_paths(value, prefix="checks"):
+    paths = []
+    if isinstance(value, dict):
+        for key, child in value.items():
+            paths.extend(false_check_paths(child, f"{prefix}.{key}"))
+    elif value is False:
+        paths.append(prefix)
+    return paths
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--ledger", default="paper/CLAIM_LEDGER.json")
+    parser.add_argument("--out", default="paper/manuscript_audit.json")
+    args = parser.parse_args()
+
+    ledger_path = absolute(args.ledger)
+    ledger = load_json(ledger_path)
+    require(ledger.get("version") == 1, "claim-ledger version drift")
+    manuscript_path = absolute(ledger["manuscript"])
+    require(os.path.isfile(manuscript_path), "manuscript is missing")
+    with open(manuscript_path) as handle:
+        manuscript = handle.read()
+
+    require(
+        manuscript.startswith(
+            "# Learning from the Unexpected: Somato-Dendritic Innovations "
+            "for Local Credit Assignment\n"
+        ),
+        "working title drift",
+    )
+    for section in EXPECTED_SECTIONS:
+        require(manuscript.count(section) == 1, f"section drift: {section}")
+    require("TODO" not in manuscript and "TBD" not in manuscript,
+            "unresolved manuscript placeholder")
+
+    manuscript_dir = os.path.dirname(manuscript_path)
+    expected_figures = {
+        os.path.normpath(os.path.join(manuscript_dir, item))
+        for item in ledger["figures"]
+    }
+    linked_figures = {
+        os.path.normpath(os.path.join(manuscript_dir, item))
+        for item in re.findall(r"!\[[^\]]*\]\(([^)]+)\)", manuscript)
+        if not item.startswith(("http://", "https://"))
+    }
+    require(
+        linked_figures == expected_figures,
+        "manuscript figure set disagrees with claim ledger",
+    )
+    for path in sorted(expected_figures):
+        require(os.path.isfile(path), f"missing manuscript figure: {path}")
+
+    source_cache = {}
+    audited_claims = []
+    for claim in ledger["numeric_claims"]:
+        source_path = absolute(claim["source"])
+        if source_path not in source_cache:
+            source_cache[source_path] = load_json(source_path)
+        value = pointer_get(source_cache[source_path], claim["pointer"])
+        rendered = render(value, claim["format"])
+        require(
+            rendered == claim["token"],
+            f"{claim['id']}: ledger token {claim['token']!r} "
+            f"does not match source rendering {rendered!r}",
+        )
+        require(
+            claim["token"] in manuscript,
+            f"{claim['id']}: audited number is absent from manuscript",
+        )
+        audited_claims.append(
+            {
+                "id": claim["id"],
+                "pointer": claim["pointer"],
+                "rendered": rendered,
+                "source": claim["source"],
+            }
+        )
+
+    gate_audits = []
+    for gate_spec in ledger["gate_statuses"]:
+        gate_path = absolute(gate_spec["source"])
+        gate = load_json(gate_path)
+        require(
+            gate.get("status") == gate_spec["expected"],
+            f"{gate_spec['source']}: gate status drift",
+        )
+        false_paths = false_check_paths(gate.get("checks", {}))
+        if gate_spec["expected"] == "passed":
+            require(not false_paths, f"{gate_spec['source']}: passed gate has false check")
+        else:
+            require(false_paths, f"{gate_spec['source']}: failed gate has no false check")
+        gate_audits.append(
+            {
+                "false_checks": false_paths,
+                "source": gate_spec["source"],
+                "status": gate["status"],
+            }
+        )
+
+    manuscript_flat = re.sub(r"\s+", " ", manuscript)
+    for sentence in ledger["required_boundary_text"]:
+        require(
+            re.sub(r"\s+", " ", sentence) in manuscript_flat,
+            f"required claim boundary absent: {sentence}",
+        )
+    for url in ledger["required_reference_urls"]:
+        require(url in manuscript, f"required primary reference absent: {url}")
+
+    source_paths = sorted(
+        {absolute(item["source"]) for item in ledger["numeric_claims"]}
+        | {absolute(item["source"]) for item in ledger["gate_statuses"]}
+    )
+    output = {
+        "audited_numeric_claims": audited_claims,
+        "figures": [
+            {
+                "path": os.path.relpath(path, ROOT),
+                "sha256": sha256(path),
+            }
+            for path in sorted(expected_figures)
+        ],
+        "gates": gate_audits,
+        "ledger": {
+            "path": os.path.relpath(ledger_path, ROOT),
+            "sha256": sha256(ledger_path),
+        },
+        "manuscript": {
+            "path": os.path.relpath(manuscript_path, ROOT),
+            "sha256": sha256(manuscript_path),
+            "word_count": len(re.findall(r"\b[\w'-]+\b", manuscript)),
+        },
+        "sources": [
+            {
+                "path": os.path.relpath(path, ROOT),
+                "sha256": sha256(path),
+            }
+            for path in source_paths
+        ],
+        "status": "passed",
+        "strict": True,
+    }
+    out_path = absolute(args.out)
+    os.makedirs(os.path.dirname(out_path), exist_ok=True)
+    with open(out_path, "w") as handle:
+        json.dump(output, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+    print(
+        json.dumps(
+            {
+                "false_gate_checks_retained": sum(
+                    len(item["false_checks"]) for item in gate_audits
+                ),
+                "numeric_claims": len(audited_claims),
+                "status": output["status"],
+                "word_count": output["manuscript"]["word_count"],
+            },
+            indent=2,
+            sort_keys=True,
+        )
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/experiments/finalize_accept.sh b/experiments/finalize_accept.sh
index 70208e9..b39a714 100755
--- a/experiments/finalize_accept.sh
+++ b/experiments/finalize_accept.sh
@@ -23,7 +23,8 @@ experiments/finalize_claims.sh
   experiments/analyze_bci_td_confirmation.py \
   experiments/oral_a_dynamic_scaling.py \
   experiments/analyze_oral_a_dynamic_scaling.py \
-  experiments/plot_resnet_confirmation.py
+  experiments/plot_resnet_confirmation.py \
+  experiments/audit_manuscript.py
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
   experiments/analyze_kp_dynamic_projection.py >/dev/null
 /home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
@@ -43,7 +44,10 @@ if [ -f results/kp_dynamic_projection_confirmation_gate.json ]; then
     .statistics.clean_minus_dynamic_one_sided_95pct_upper_points <= 2.5 and
     .statistics.dynamic_early_alignment_mean >= 0.90 and
     .statistics.dynamic_mac_ratio_to_bp <= 1.34 and
-    .statistics.dynamic_logical_task_loss_queries == [0]
+    .statistics.dynamic_logical_task_loss_queries == [0] and
+    .statistics.dynamic_neutral_observations_per_ordinary_example == [1] and
+    .statistics.dynamic_peak_allocated_gib_mean <= 2.5 and
+    .statistics.dynamic_wall_ratio_to_clean_kp_mean > 1
   ' results/figs/figure4_resnet_confirmation_manifest.json >/dev/null
 fi
 if [ -f results/bci_td_dev_gate.json ]; then
@@ -66,6 +70,16 @@ fi
   experiments/audit_native_baselines.py \
   --results results/native \
   --output results/native_baseline_audit.md
+/home/yurenh2/miniconda3/envs/ep_pascal/bin/python3 \
+  experiments/audit_manuscript.py >/dev/null
+jq -e '
+  .strict == true and
+  .status == "passed" and
+  (.audited_numeric_claims | length) == 34 and
+  (.gates | map(select(.status == "passed")) | length) == 1 and
+  (.gates | map(select(.status == "failed")) | length) == 1 and
+  (.gates[] | select(.status == "failed") | .false_checks | length) == 7
+' paper/manuscript_audit.json >/dev/null
 
 git diff --check
 echo "accept-bar mechanical audits passed"
diff --git a/experiments/plot_resnet_confirmation.py b/experiments/plot_resnet_confirmation.py
index efd0d48..99a55d4 100644
--- a/experiments/plot_resnet_confirmation.py
+++ b/experiments/plot_resnet_confirmation.py
@@ -483,6 +483,7 @@ def main():
         "statistics": {
             "clean_kp_test_accuracy_mean": float(clean.mean()),
             "dynamic_test_accuracy_mean": float(dynamic.mean()),
+            "dynamic_test_accuracy_min": float(dynamic.min()),
             "dynamic_minus_clean_points_mean": float(
                 (dynamic - clean).mean() * 100
             ),
@@ -499,6 +500,22 @@ def main():
                     for seed in EXPECTED_SEEDS
                 )
             ),
+            "dynamic_peak_allocated_gib_mean": float(
+                statistics.mean(
+                    records[(seed, "kp_traffic")]["hardware"][
+                        "peak_memory_allocated_bytes"
+                    ]
+                    / (1024 ** 3)
+                    for seed in EXPECTED_SEEDS
+                )
+            ),
+            "dynamic_wall_ratio_to_clean_kp_mean": float(
+                statistics.mean(
+                    records[(seed, "kp_traffic")]["timing"]["total_timed_wall_s"]
+                    / records[(seed, "kp")]["timing"]["total_timed_wall_s"]
+                    for seed in EXPECTED_SEEDS
+                )
+            ),
             "dynamic_logical_task_loss_queries": sorted(
                 {
                     records[(seed, "kp_traffic")]["work"][
@@ -507,6 +524,17 @@ def main():
                     for seed in EXPECTED_SEEDS
                 }
             ),
+            "dynamic_neutral_observations_per_ordinary_example": sorted(
+                {
+                    records[(seed, "kp_traffic")]["counters"][
+                        "neutral_projection_examples"
+                    ]
+                    / records[(seed, "kp_traffic")]["counters"][
+                        "ordinary_examples"
+                    ]
+                    for seed in EXPECTED_SEEDS
+                }
+            ),
         },
         "sources": [
             {"path": args.gate, "sha256": sha256(args.gate)}