diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_contrastive_bias_b1.py | 35 | ||||
| -rw-r--r-- | experiments/plot_contrastive_bias_b1.py | 80 |
2 files changed, 107 insertions, 8 deletions
diff --git a/experiments/analyze_contrastive_bias_b1.py b/experiments/analyze_contrastive_bias_b1.py index 527b2ba..d750abf 100644 --- a/experiments/analyze_contrastive_bias_b1.py +++ b/experiments/analyze_contrastive_bias_b1.py @@ -19,16 +19,30 @@ def cell_path(cell_id): def valid_record(record, cell): history = record.get("history") or {} - return ( + base_valid = ( record.get("status") == "completed" and record.get("cell_id") == cell["cell_id"] and record.get("kind") == cell["kind"] and record.get("rule") == cell["rule"] and record.get("ratio") == cell["ratio"] - and history.get("finite") is True - and history.get("epochs_completed") == 20 and math.isnan(float(history.get("test_accuracy", float("nan")))) ) + if not base_valid: + return False + if history.get("finite") is True: + return history.get("epochs_completed") == 20 + # The protocol explicitly retains a differential raw condition that + # becomes nonfinite. No other condition may use this exception. + if not (cell["rule"] == "raw" and cell["kind"] in ("fixed", "activity")): + return False + completed = history.get("epochs_completed") + curves = history.get("curves") or {} + losses = list(curves.get("train_loss", [])) + list(curves.get("val_loss", [])) + return ( + isinstance(completed, int) and 1 <= completed <= 20 + and len(losses) >= 2 * completed + and any(not math.isfinite(float(value)) for value in losses) + ) def metric_max(record, key): @@ -69,11 +83,12 @@ def main(): candidates = [] table = [] for ratio in (0.25, 1.0, 4.0): - tag = rate_tag = f"{ratio:g}".replace(".", "p") + tag = f"{ratio:g}".replace(".", "p") row = {"ratio": ratio} for rule in ("raw", "innovation", "oracle"): record = records[f"activity-r{tag}-{rule}"] row[rule] = float(record["history"]["final_validation_accuracy"]) + row[rule + "_finite"] = bool(record["history"]["finite"]) row["raw_degradation"] = clean_acc - row["raw"] row["innovation_clean_gap"] = abs(row["innovation"] - clean_acc) row["innovation_oracle_gap"] = abs(row["innovation"] - row["oracle"]) @@ -81,7 +96,8 @@ def main(): records[f"activity-r{tag}-innovation"], "post_bias_raw_bias_rms_ratio") row["passes"] = ( - row["raw_degradation"] >= 5.0 + (not row["raw_finite"] or row["raw_degradation"] >= 5.0) + and row["innovation_finite"] and row["oracle_finite"] and row["innovation_clean_gap"] <= 2.0 and row["innovation_oracle_gap"] <= 1.0 and row["innovation_post_bias_ratio_max"] <= 1e-3 @@ -101,16 +117,19 @@ def main(): registry_values = {row["registry_sha256"] for row in records.values()} checks["single_source_lock"] = len(source_values) == 1 checks["single_registry_lock"] = len(registry_values) == 1 + gate = "pass" if all(checks.values()) else "fail" report = { - "stage": "contrastive_bias_b1", "gate": ( - "pass" if all(checks.values()) else "fail"), + "stage": "contrastive_bias_b1", "gate": gate, "checks": checks, "clean_final_validation_accuracy": clean_acc, "common_final_validation_accuracy": common_acc, "common_maximum_difference_relative_error": common_error, "innovation_maximum_post_bias_ratio": innovation_post_max, "predictor_maximum_instruction_observations": predictor_instruction_max, "activity_table": table, - "selected_confirmation_ratio": max(candidates) if candidates else None, + "largest_core_passing_activity_ratio": ( + max(candidates) if candidates else None), + "selected_confirmation_ratio": ( + max(candidates) if candidates and gate == "pass" else None), "num_expected_records": 17, "num_audited_records": len(records), "source": clean["source"], "registry_sha256": clean["registry_sha256"], } diff --git a/experiments/plot_contrastive_bias_b1.py b/experiments/plot_contrastive_bias_b1.py new file mode 100644 index 0000000..5fc70d1 --- /dev/null +++ b/experiments/plot_contrastive_bias_b1.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Plot the audited B1 activity-bias result without rerunning analysis.""" +import argparse +import json +from pathlib import Path + +import matplotlib.pyplot as plt +import numpy as np + + +ROOT = Path(__file__).resolve().parents[1] +DEFAULT_GATE = ROOT / "results" / "contrastive_bias" / "b1_gate.json" +DEFAULT_OUT = ROOT / "results" / "contrastive_bias" / "b1_summary" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--gate", type=Path, default=DEFAULT_GATE) + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + args = parser.parse_args() + with open(args.gate, encoding="utf-8") as handle: + report = json.load(handle) + rows = report["activity_table"] + ratios = np.arange(len(rows)) + labels = [f"{row['ratio']:g}×" for row in rows] + clean = report["clean_final_validation_accuracy"] + + plt.rcParams.update({ + "font.family": "DejaVu Sans", "font.size": 9, + "axes.spines.top": False, "axes.spines.right": False, + }) + figure, axes = plt.subplots(1, 2, figsize=(7.2, 2.65)) + + ax = axes[0] + ax.axhline(clean, color="#6b747b", linestyle="--", linewidth=1.2, + label=f"clean DP: {clean:.2f}") + ax.plot(ratios, [row["raw"] for row in rows], "X-", color="#c23b3b", + linewidth=1.5, markersize=7, label="raw (nonfinite at epoch 1)") + ax.plot(ratios, [row["innovation"] for row in rows], "o-", + color="#0878a8", linewidth=1.8, markersize=5, label="innovation") + ax.plot(ratios, [row["oracle"] for row in rows], "s-", + color="#263640", linewidth=1.5, markersize=4.5, label="oracle") + ax.set_xticks(ratios, labels) + ax.set_ylim(0, 78) + ax.set_xlabel("activity-dependent bias / clean teaching RMS") + ax.set_ylabel("final validation accuracy (%)") + ax.set_title("a Task result", loc="left", fontweight="bold") + ax.legend(frameon=False, fontsize=7.5, loc="lower right") + + ax = axes[1] + width = 0.23 + ax.bar(ratios - width, [1, 1, 1], width, color="#c23b3b", label="raw") + ax.bar(ratios, [20, 20, 20], width, color="#0878a8", label="innovation") + ax.bar(ratios + width, [20, 20, 20], width, color="#263640", label="oracle") + ax.set_xticks(ratios, labels) + ax.set_ylim(0, 22) + ax.set_yticks((0, 5, 10, 15, 20)) + ax.set_xlabel("activity-dependent bias / clean teaching RMS") + ax.set_ylabel("epochs completed before nonfinite loss") + ax.set_title("b Stability", loc="left", fontweight="bold") + ax.legend(frameon=False, fontsize=7.5, loc="lower right") + ax.text( + 0.02, 0.95, + "innovation residual bias ≤ 8.96×10⁻⁸\npredictor label observations = 0", + transform=ax.transAxes, va="top", fontsize=7.5, + ) + + figure.suptitle( + "Dual Prop under neuron-specific activity bias", + x=0.08, ha="left", fontsize=12, fontweight="bold", + ) + figure.tight_layout(rect=(0, 0, 1, 0.94)) + args.out.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(args.out.with_suffix(".pdf"), bbox_inches="tight") + figure.savefig(args.out.with_suffix(".png"), dpi=220, bbox_inches="tight") + plt.close(figure) + + +if __name__ == "__main__": + main() |