diff options
Diffstat (limited to 'experiments/resnet_crossover_smoke.py')
| -rw-r--r-- | experiments/resnet_crossover_smoke.py | 85 |
1 files changed, 84 insertions, 1 deletions
diff --git a/experiments/resnet_crossover_smoke.py b/experiments/resnet_crossover_smoke.py index aadede5..8cdcb0c 100644 --- a/experiments/resnet_crossover_smoke.py +++ b/experiments/resnet_crossover_smoke.py @@ -8,7 +8,11 @@ import torch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from sdil.conv import CIFARLocalResNet -from sdil.conv_crossover import CIFARDualPropResNet, CIFARPEPITAResNet +from sdil.conv_crossover import ( + CIFARDualPropResNet, + CIFARForwardForwardResNet, + CIFARPEPITAResNet, +) def relative_error(actual, expected): @@ -167,9 +171,88 @@ def audit_pepita(): } +def audit_forward_forward(): + common = dict( + depth=8, base_width=2, seed=91, normalization="batchnorm", + residual_scale=1.0, dtype=torch.float64) + reference = CIFARLocalResNet(**common) + net = CIFARForwardForwardResNet( + **common, threshold=2.0, learning_rate=0.03, + score_from_layer=1) + if net.n_forward_parameters != reference.n_forward_parameters: + raise AssertionError("Forward-Forward changed forward parameter count") + generator = torch.Generator().manual_seed(92) + image = torch.randn(4, 3, 32, 32, generator=generator, + dtype=torch.float64) + labels = torch.tensor([1, 2, 6, 9]) + negative_labels = (labels + 3) % 10 + target = 1 + with torch.no_grad(): + positive = net.ff_forward( + net.ff_overlay(image, labels), + training=True, update_stats=False) + negative = net.ff_forward( + net.ff_overlay(image, negative_labels), + training=True, update_stats=False) + positive_output, negative_output = net._ff_local_outputs( + target, positive, negative) + axes = tuple(range(1, positive_output.ndim)) + positive_goodness = positive_output.square().mean(dim=axes) + negative_goodness = negative_output.square().mean(dim=axes) + explicit_loss = ( + torch.nn.functional.softplus(-positive_goodness + net.ff_threshold) + + torch.nn.functional.softplus( + negative_goodness - net.ff_threshold) + ).mean() + parameters = ( + net.W + net.gamma + net.beta + [net.W_out, net.b_out]) + before = [parameter.detach().clone() for parameter in parameters] + metrics = net.ff_train_layer( + target, image, labels, learning_rate=0.03, + negative_labels=negative_labels) + changed = [ + not torch.equal(old, parameter.detach()) + for old, parameter in zip(before, parameters)] + target_indices = {target} + if net.normalization == "batchnorm": + target_indices.update({ + len(net.W) + target, + len(net.W) + len(net.gamma) + target, + }) + non_target_changes = [ + index for index, value in enumerate(changed) + if value and index not in target_indices] + non_target_gradients = [ + index for index, parameter in enumerate(parameters) + if index not in target_indices and parameter.grad is not None] + scores = net.ff_candidate_scores(image) + loss_error = abs(metrics["loss"] - float(explicit_loss.detach())) + if (loss_error >= 1e-12 or non_target_changes + or non_target_gradients or not any( + changed[index] for index in target_indices) + or scores.shape != (4, 10) + or not torch.isfinite(scores).all()): + raise AssertionError({ + "loss_error": loss_error, + "non_target_changes": non_target_changes, + "non_target_gradients": non_target_gradients, + "changed": changed, + "score_shape": list(scores.shape), + }) + return { + "local_objective_absolute_error": loss_error, + "non_target_parameter_changes": len(non_target_changes), + "non_target_gradients": len(non_target_gradients), + "candidate_score_shape": list(scores.shape), + "matched_forward_parameter_count": net.n_forward_parameters, + "uses_only_target_layer_autograd": True, + } + + def main(): print(json.dumps({ "dualprop": audit_dualprop(), + "forward_forward": audit_forward_forward(), "pepita": audit_pepita(), }, indent=2, sort_keys=True)) |