diff options
Diffstat (limited to 'experiments/resnet_crossover_smoke.py')
| -rw-r--r-- | experiments/resnet_crossover_smoke.py | 80 |
1 files changed, 80 insertions, 0 deletions
diff --git a/experiments/resnet_crossover_smoke.py b/experiments/resnet_crossover_smoke.py index 8cdcb0c..6d18c7f 100644 --- a/experiments/resnet_crossover_smoke.py +++ b/experiments/resnet_crossover_smoke.py @@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from sdil.conv import CIFARLocalResNet from sdil.conv_crossover import ( CIFARDualPropResNet, + CIFAREquilibriumPropResNet, CIFARForwardForwardResNet, CIFARPEPITAResNet, ) @@ -249,9 +250,88 @@ def audit_forward_forward(): } +def audit_equilibrium_propagation(): + common = dict( + depth=8, base_width=2, seed=101, normalization="batchnorm", + residual_scale=1.0, dtype=torch.float64) + reference = CIFARLocalResNet(**common) + net = CIFAREquilibriumPropResNet( + **common, ep_beta=0.5, dt=0.5, free_steps=8, + nudge_steps=2, random_beta_sign=False) + generator = torch.Generator().manual_seed(102) + image = torch.randn(3, 3, 32, 32, generator=generator, + dtype=torch.float64) + labels = torch.tensor([0, 5, 9]) + one_hot = torch.nn.functional.one_hot(labels, 10).to(torch.float64) + with torch.no_grad(): + free = net.ep_settle( + image, one_hot, beta=0.0, steps=net.ep_free_steps) + signed_beta = net.ep_beta + nudged = net.ep_settle( + image, one_hot, beta=signed_beta, steps=net.ep_nudge_steps, + initial_states=free) + parameters = ( + net.W + net.gamma + net.beta + [net.W_out, net.b_out]) + for parameter in parameters: + parameter.requires_grad_(True) + + def phase_phi(states): + clipped_image = net.ep_rho(image) + hidden = [net.ep_rho(value.detach()) for value in states[:-1]] + value = image.new_zeros(()) + for index in range(net.n_hidden): + prediction, _ = net._node_prediction( + index, hidden, clipped_image) + value += torch.sum( + prediction * net.ep_rho(states[index].detach())) + features = hidden[-1].mean(dim=(2, 3)) + output_prediction = features @ net.W_out.t() + net.b_out + value += torch.sum( + output_prediction * net.ep_rho(states[-1].detach())) + return value + + objective = ( + phase_phi(free) - phase_phi(nudged) + ) / (signed_beta * image.shape[0]) + gradients = torch.autograd.grad(objective, parameters) + (directions, gamma_directions, beta_directions, + output_weight, output_bias) = net.ep_ascent_directions( + image, free, nudged, signed_beta) + actual = ( + directions + gamma_directions + beta_directions + + [output_weight, output_bias]) + errors = [ + relative_error(direction, -gradient) + for direction, gradient in zip(actual, gradients)] + for parameter in parameters: + parameter.requires_grad_(False) + state_bounds = [ + (float(value.min()), float(value.max())) + for value in free + nudged] + if (net.n_forward_parameters != reference.n_forward_parameters + or max(errors) >= 2e-12 + or any(low < 0 or high > 1 for low, high in state_bounds) + or not all(torch.isfinite(value).all() + for value in free + nudged)): + raise AssertionError({ + "direction_errors": errors, + "state_bounds": state_bounds, + }) + return { + "contrastive_direction_max_relative_error": max(errors), + "matched_forward_parameter_count": net.n_forward_parameters, + "free_steps": net.ep_free_steps, + "nudge_steps": net.ep_nudge_steps, + "state_minimum": min(low for low, _ in state_bounds), + "state_maximum": max(high for _, high in state_bounds), + "uses_reverse_task_loss_graph": False, + } + + def main(): print(json.dumps({ "dualprop": audit_dualprop(), + "equilibrium_propagation": audit_equilibrium_propagation(), "forward_forward": audit_forward_forward(), "pepita": audit_pepita(), }, |