diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_coupled_ladder_scaling.py | 113 |
1 files changed, 113 insertions, 0 deletions
diff --git a/experiments/analyze_coupled_ladder_scaling.py b/experiments/analyze_coupled_ladder_scaling.py index d93b056..1851094 100644 --- a/experiments/analyze_coupled_ladder_scaling.py +++ b/experiments/analyze_coupled_ladder_scaling.py @@ -78,6 +78,11 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--output-figure", type=Path, default=Path("results/figs/figure_clln_scaling_pilot")) + parser.add_argument( + "--output-resource-figure", + type=Path, + help="optional base path for the censored cost-to-target figure", + ) parser.add_argument("--bootstrap-replicates", type=int, default=20000) parser.add_argument("--bootstrap-seed", type=int, default=20260829) parser.add_argument( @@ -718,6 +723,107 @@ def plot_figure(path: Path, analysis: dict) -> None: plt.close(figure) +def plot_resource_figure(path: Path, analysis: dict) -> None: + mpl.rcParams.update({ + "font.family": "DejaVu Sans", + "font.size": 8.5, + "axes.labelsize": 9, + "axes.titlesize": 9.5, + "legend.fontsize": 7.5, + "xtick.labelsize": 8, + "ytick.labelsize": 8, + "axes.spines.top": False, + "axes.spines.right": False, + "svg.fonttype": "none", + "pdf.fonttype": 42, + "figure.facecolor": "white", + "axes.facecolor": "white", + }) + sizes = analysis["sizes"] + edges = np.asarray([ + analysis["summaries"][str(side)]["learnable_edges"] + for side in sizes + ]) + edge_labels = [ + f"{value / 1000.0:.1f}k" if value >= 1000 else str(value) + for value in edges + ] + panels = ( + ( + "restricted_updates_to_stable_zero_error", + "Local updates to stable zero error", + "restricted_update_scaling", + ), + ( + "restricted_teaching_signal_reads_to_stable_zero_error", + "Local scalar reads to stable zero error", + "teaching_signal_read_scaling", + ), + ) + figure, axes = plt.subplots(1, 2, figsize=(7.4, 3.1)) + for panel_index, (metric, ylabel, scaling_key) in enumerate(panels): + axis = axes[panel_index] + for method in METHOD_ORDER: + means = np.asarray([ + analysis["summaries"][str(side)]["methods"][method][metric][ + "mean" + ] + for side in sizes + ]) + intervals = np.asarray([ + analysis["summaries"][str(side)]["methods"][method][metric][ + "task_bootstrap_95ci" + ] + for side in sizes + ]) + axis.errorbar( + edges, + means, + yerr=np.vstack((means - intervals[:, 0], intervals[:, 1] - means)), + linewidth=1.7 if method == "sdil" else 1.15, + markersize=4.5, + capsize=2.0, + label=DISPLAY[method], + **STYLE[method], + ) + ratio = analysis[scaling_key]["largest_size"]["sdil_to_static_ratio"] + axis.set_title( + f"({chr(97 + panel_index)}) Censored cost to target\n" + f"SDIL/static = {ratio:.2f} at {edges[-1]:,} edges" + ) + axis.set_xscale("log", base=2) + axis.set_yscale("log") + axis.set_xticks(edges, edge_labels) + axis.set_xlabel("Learnable edges") + axis.set_ylabel(ylabel) + axis.grid(axis="y", color="#D9D9D9", linewidth=0.55, alpha=0.8) + handles, labels = axes[0].get_legend_handles_labels() + figure.legend( + handles, + labels, + loc="upper center", + ncol=5, + frameon=False, + bbox_to_anchor=(0.5, 1.02), + ) + figure.text( + 0.995, + 0.005, + "Non-successes receive the 600-epoch horizon; SDIL counts task + " + "neutral reads; static calibration includes 16 edgewise reads.", + ha="right", + va="bottom", + fontsize=7, + color="#666666", + ) + figure.tight_layout(rect=(0.0, 0.055, 1.0, 0.90), w_pad=2.0) + path.parent.mkdir(parents=True, exist_ok=True) + figure.savefig(path.with_suffix(".svg"), bbox_inches="tight") + figure.savefig(path.with_suffix(".pdf"), bbox_inches="tight") + figure.savefig(path.with_suffix(".png"), dpi=240, bbox_inches="tight") + plt.close(figure) + + def main() -> None: args = parse_args() source_paths = list(args.core) + list(args.baselines) @@ -736,6 +842,12 @@ def main() -> None: args.output_analysis.write_text(json.dumps(analysis, indent=2) + "\n") write_csv(args.output_csv, analysis) plot_figure(args.output_figure, analysis) + resource_figure = args.output_resource_figure + if resource_figure is None: + resource_figure = args.output_figure.with_name( + args.output_figure.name + "_resources" + ) + plot_resource_figure(resource_figure, analysis) print(json.dumps({ "excess_error_scaling": analysis["excess_error_scaling"], "excess_error_auc_scaling": analysis["excess_error_auc_scaling"], @@ -752,6 +864,7 @@ def main() -> None: print(f"wrote {args.output_analysis}") print(f"wrote {args.output_csv}") print(f"wrote {args.output_figure}.svg/.pdf/.png") + print(f"wrote {resource_figure}.svg/.pdf/.png") if __name__ == "__main__": |