diff options
Diffstat (limited to 'experiments')
| -rw-r--r-- | experiments/analyze_coupled_ladder_scaling.py | 140 |
1 files changed, 128 insertions, 12 deletions
diff --git a/experiments/analyze_coupled_ladder_scaling.py b/experiments/analyze_coupled_ladder_scaling.py index 1c8a926..0cbbb40 100644 --- a/experiments/analyze_coupled_ladder_scaling.py +++ b/experiments/analyze_coupled_ladder_scaling.py @@ -45,6 +45,18 @@ FROZEN_LEARNING_TIMES = { 24: 0.03, 32: 0.01, } +PRIMARY_METRICS = ( + "classification_error", + "classification_error_auc", + "reached_stable_zero_error", +) +RESOURCE_METRICS = ( + "restricted_epochs_to_stable_zero_error", + "restricted_updates_to_stable_zero_error", + "restricted_edge_updates_to_stable_zero_error", + "restricted_teaching_signal_reads_to_stable_zero_error", + "restricted_learning_time_to_stable_zero_seconds", +) def parse_args() -> argparse.Namespace: @@ -158,13 +170,26 @@ def task_cluster_values( }) values = [] for task_index in task_indices: - device_values = [ - record["methods"][method][metric] - for record in records - if record["side"] == side - and record["task_index"] == task_index - and method in record["methods"] - ] + device_values = [] + for record in records: + if ( + record["side"] != side + or record["task_index"] != task_index + or method not in record["methods"] + ): + continue + method_record = record["methods"][method] + if metric == "restricted_teaching_signal_reads_to_stable_zero_error": + value = method_record[ + "restricted_edge_updates_to_stable_zero_error" + ] + if method == "sdil": + value *= 2 + elif method == "constant": + value += 16 * record["learnable_edges"] + else: + value = method_record[metric] + device_values.append(value) values.append(float(np.mean(device_values))) return np.asarray(task_indices), np.asarray(values) @@ -274,6 +299,82 @@ def metric_slope_bootstrap( } +def resource_exponent_bootstrap( + records: list[dict], + sizes: list[int], + edges: np.ndarray, + *, + metric: str, + replicates: int, + seed: int, +) -> dict: + """Fit paired log-log resource exponents for static calibration and SDIL.""" + _, constant = task_size_matrix(records, sizes, "constant", metric) + _, sdil = task_size_matrix(records, sizes, "sdil", metric) + x = np.log10(edges) + + def exponent(matrix: np.ndarray) -> float: + means = np.mean(matrix, axis=0) + if np.any(means <= 0.0): + raise ValueError(f"resource metric {metric} must be positive") + return float(np.polyfit(x, np.log10(means), 1)[0]) + + point_constant = exponent(constant) + point_sdil = exponent(sdil) + point_difference = point_constant - point_sdil + point_reduction = ( + 100.0 * point_difference / point_constant + if point_constant > 0.0 else None + ) + rng = np.random.default_rng(seed) + constant_exponents = [] + sdil_exponents = [] + differences = [] + reductions = [] + for _ in range(replicates): + sample = rng.integers(0, len(constant), size=len(constant)) + constant_exponent = exponent(constant[sample]) + sdil_exponent = exponent(sdil[sample]) + constant_exponents.append(constant_exponent) + sdil_exponents.append(sdil_exponent) + differences.append(constant_exponent - sdil_exponent) + if constant_exponent > 1e-12: + reductions.append( + 100.0 + * (constant_exponent - sdil_exponent) + / constant_exponent + ) + + def interval(values: list[float]) -> list[float]: + return [ + float(value) for value in np.percentile(values, (2.5, 97.5)) + ] + + constant_largest = float(np.mean(constant[:, -1])) + sdil_largest = float(np.mean(sdil[:, -1])) + return { + "x_axis": "log10 learnable edges", + "y_axis": f"log10 {metric}", + "metric": metric, + "static_calibration_exponent": point_constant, + "static_calibration_exponent_95ci": interval(constant_exponents), + "sdil_exponent": point_sdil, + "sdil_exponent_95ci": interval(sdil_exponents), + "paired_exponent_difference": point_difference, + "paired_exponent_difference_95ci": interval(differences), + "relative_exponent_reduction_percent": point_reduction, + "relative_exponent_reduction_95ci": ( + interval(reductions) if reductions else None + ), + "largest_size": { + "learnable_edges": int(edges[-1]), + "static_calibration_mean": constant_largest, + "sdil_mean": sdil_largest, + "sdil_to_static_ratio": sdil_largest / constant_largest, + }, + } + + def trace_summary( records: list[dict], side: int, method: str ) -> list[dict]: @@ -343,11 +444,7 @@ def build_analysis( side_summary = {} for method in METHOD_ORDER: metrics = {} - for metric in ( - "classification_error", - "classification_error_auc", - "reached_stable_zero_error", - ): + for metric in PRIMARY_METRICS + RESOURCE_METRICS: _, values = task_cluster_values( records, side, method, metric) mean, interval = bootstrap_mean( @@ -407,6 +504,22 @@ def build_analysis( replicates=replicates, seed=seed + 3, ), + "restricted_update_scaling": resource_exponent_bootstrap( + records, + sizes, + edges, + metric="restricted_updates_to_stable_zero_error", + replicates=replicates, + seed=seed + 4, + ), + "teaching_signal_read_scaling": resource_exponent_bootstrap( + records, + sizes, + edges, + metric="restricted_teaching_signal_reads_to_stable_zero_error", + replicates=replicates, + seed=seed + 5, + ), "largest_grid_learning_curves": { method: trace_summary(records, sizes[-1], method) for method in METHOD_ORDER @@ -628,6 +741,9 @@ def main() -> None: "excess_error_auc_scaling": analysis["excess_error_auc_scaling"], "excess_stable_failure_scaling": analysis[ "excess_stable_failure_scaling"], + "restricted_update_scaling": analysis["restricted_update_scaling"], + "teaching_signal_read_scaling": analysis[ + "teaching_signal_read_scaling"], "largest_side": analysis["sizes"][-1], "largest_side_methods": analysis["summaries"][ str(analysis["sizes"][-1]) |