diff options
| -rw-r--r-- | sdil/core.py | 15 |
1 files changed, 11 insertions, 4 deletions
diff --git a/sdil/core.py b/sdil/core.py index 048408e..3115d46 100644 --- a/sdil/core.py +++ b/sdil/core.py @@ -338,6 +338,16 @@ def sdil_step(net, x, y, y_onehot, cfg, step, prev_error=None): r_list.append(r) a_list.append(a) + # Calibrate A against the same network state that produced r and c. + # Measuring q after mutating W would pair a post-update causal target + # with a pre-update prediction, introducing an avoidable stale-target + # error (especially at large learning rates). + did_pert = cfg.learn_A and (step % cfg.pert_every == 0) + qs = None + if did_pert: + qs = node_perturbation_targets( + net, x, y, sigma=cfg.pert_sigma, n_dirs=cfg.pert_ndirs) + # ================= forward weight updates ================= # hidden layers: three-factor local rule for l in range(net.L - 1): @@ -362,10 +372,7 @@ def sdil_step(net, x, y, y_onehot, cfg, step, prev_error=None): _apply(net, net.L - 1, dWL, dbL, cfg) # ================= apical vectorizer A via node perturbation ======= - did_pert = False - if cfg.learn_A and (step % cfg.pert_every == 0): - did_pert = True - qs = node_perturbation_targets(net, x, y, sigma=cfg.pert_sigma, n_dirs=cfg.pert_ndirs) + if did_pert: for l in range(net.L - 1): dA = (qs[l] - r_list[l]).t() @ c / B # (n_l, n_classes) net.A[l] += cfg.eta_A * dA |