From 1e4fbaf8e773509798387d19b806f6daf38f8b5c Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Wed, 22 Jul 2026 13:47:35 -0500 Subject: baseline: add residual response mirroring --- experiments/conv_local_smoke.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'experiments/conv_local_smoke.py') diff --git a/experiments/conv_local_smoke.py b/experiments/conv_local_smoke.py index 7d24154..4e388a0 100644 --- a/experiments/conv_local_smoke.py +++ b/experiments/conv_local_smoke.py @@ -14,6 +14,7 @@ from sdil.conv import (CIFARHierarchicalFAResNet, CIFARLocalResNet, conv_hierarchical_step, conv_local_step, hierarchical_mirror_observations, hierarchical_parameter_subspace_calibration, + normalized_residual_mirror_update, normalized_response_mirror_update, simultaneous_conv_node_perturbation, vectorizer_subspace_apical_calibration) @@ -794,6 +795,38 @@ def normalized_response_mirror_checks(): independence_error = max(float((a - b).abs().max()) for a, b in zip( left.Q[1:] + [left.R_out], right.Q[1:] + [right.R_out])) assert independence_error == 0.0 + + # Residual-response LMS has a per-observation exact fixed point: its update + # is zero, not merely zero in expectation, when Q/R match the forward maps. + fixed = CIFARHierarchicalFAResNet( + depth=8, base_width=2, seed=125, dtype=torch.float64) + fixed.Q = [value.clone() for value in fixed.W] + fixed.R_out.copy_(-fixed.W_out.t()) + fixed_observations = hierarchical_mirror_observations( + fixed, batch_size=2, generator=torch.Generator().manual_seed(126)) + fixed_metrics = normalized_residual_mirror_update( + fixed, fixed_observations, eta=1.0) + assert fixed_metrics["mirror_update_rms"] < 1e-14 + assert fixed_metrics["mirror_response_residual_fraction"] < 1e-14 + + residual_left = CIFARHierarchicalFAResNet( + depth=8, base_width=2, seed=127, dtype=torch.float64) + residual_right = CIFARHierarchicalFAResNet( + depth=8, base_width=2, seed=127, dtype=torch.float64) + residual_observations = hierarchical_mirror_observations( + residual_left, batch_size=2, + generator=torch.Generator().manual_seed(128)) + for value in residual_right.W + [residual_right.W_out]: + value.normal_(generator=torch.Generator().manual_seed(value.numel() + 1)) + normalized_residual_mirror_update( + residual_left, residual_observations, eta=0.2) + normalized_residual_mirror_update( + residual_right, residual_observations, eta=0.2) + residual_independence_error = max(float((a - b).abs().max()) + for a, b in zip( + residual_left.Q[1:] + [residual_left.R_out], + residual_right.Q[1:] + [residual_right.R_out])) + assert residual_independence_error == 0.0 return { "mirror_estimate_mean_forward_cosine": sum(cosines) / len(cosines), "mirror_estimate_min_forward_cosine": min(cosines), @@ -801,6 +834,12 @@ def normalized_response_mirror_checks(): "mirror_estimate_max_norm_ratio": max(norm_ratios), "mirror_update_forward_parameter_independence_error": independence_error, "mirror_update_rms": metrics["mirror_update_rms"], + "residual_mirror_exact_fixed_point_update_rms": fixed_metrics[ + "mirror_update_rms"], + "residual_mirror_exact_fixed_point_fraction": fixed_metrics[ + "mirror_response_residual_fraction"], + "residual_mirror_forward_parameter_independence_error": ( + residual_independence_error), } -- cgit v1.2.3