diff options
Diffstat (limited to 'experiments')
| -rwxr-xr-x | experiments/narrow_depth_sweep.sh | 6 | ||||
| -rw-r--r-- | experiments/run.py | 2 | ||||
| -rw-r--r-- | experiments/smoke.py | 8 |
3 files changed, 13 insertions, 3 deletions
diff --git a/experiments/narrow_depth_sweep.sh b/experiments/narrow_depth_sweep.sh index 8f62a68..3e4e6b2 100755 --- a/experiments/narrow_depth_sweep.sh +++ b/experiments/narrow_depth_sweep.sh @@ -3,7 +3,7 @@ # # Usage: # bash experiments/narrow_depth_sweep.sh \ -# <gpu> "<depths>" "<modes>" <width> <seed> <epochs> <prefix> +# <gpu> "<depths>" "<modes>" <width> <seed> <epochs> <prefix> [pert_mode] [n_dirs] # # Example: # bash experiments/narrow_depth_sweep.sh 5 "30 60" "bp dfa sdil" 64 0 5 pilot_jac @@ -18,6 +18,8 @@ WIDTH="${4:-64}" SEED="${5:-0}" EPOCHS="${6:-5}" PREFIX="${7:-narrow}" +PERT_MODE="${8:-layerwise}" +N_DIRS="${9:-8}" export CUDA_VISIBLE_DEVICES="$GPU" export OMP_NUM_THREADS=2 @@ -41,7 +43,7 @@ run_one() { "$PY" experiments/run.py \ --mode "$mode" --dataset cifar10 --depth "$depth" --width "$WIDTH" \ --act tanh --residual 1 --epochs "$EPOCHS" --seed "$SEED" \ - --eta "$eta" --eta_A 0.02 --pert_ndirs 8 \ + --eta "$eta" --eta_A 0.02 --pert_ndirs "$N_DIRS" --pert_mode "$PERT_MODE" \ --log_every 500 --probe_bs 256 --tag "$tag" --outdir results \ > "$log" 2>&1 grep -h DONE "$log" diff --git a/experiments/run.py b/experiments/run.py index bb2d151..eb9f356 100644 --- a/experiments/run.py +++ b/experiments/run.py @@ -61,6 +61,7 @@ def build(args, device): use_residual=bool(args.use_residual), learn_A=bool(args.learn_A), learn_P=bool(args.learn_P), pert_sigma=args.pert_sigma, pert_every=args.pert_every, pert_ndirs=args.pert_ndirs, + pert_mode=args.pert_mode, momentum=args.momentum, settle_steps=args.settle_steps, kappa=args.kappa, feedback=args.feedback, p_update_on_neutral=bool(args.p_neutral), @@ -169,6 +170,7 @@ def get_args(): p.add_argument("--pert_sigma", type=float, default=1e-2) p.add_argument("--pert_every", type=int, default=4) p.add_argument("--pert_ndirs", type=int, default=4) + p.add_argument("--pert_mode", default="layerwise", choices=["layerwise", "simultaneous"]) p.add_argument("--use_residual", type=int, default=1) p.add_argument("--learn_A", type=int, default=1) p.add_argument("--learn_P", type=int, default=1) diff --git a/experiments/smoke.py b/experiments/smoke.py index 8196d4f..f440f7f 100644 --- a/experiments/smoke.py +++ b/experiments/smoke.py @@ -7,7 +7,8 @@ import torch sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from sdil.core import (SDILNet, SDILConfig, sdil_step, - node_perturbation_targets, neutral_p_update) + node_perturbation_targets, simultaneous_node_perturbation_targets, + neutral_p_update) from sdil.baselines import dfa_config from sdil import probes from sdil.data import get_dataset, onehot @@ -104,6 +105,11 @@ def main(): print(f"CHECK1 node-pert n_dirs={ndirs:2d} cos(q,-g) per layer = " + " ".join(f"{c:+.3f}" for c in cs)) assert all(c > 0 for c in cs), "node perturbation q should align with -grad" + qs = simultaneous_node_perturbation_targets(net, x, y, sigma=1e-2, n_dirs=32) + cs = [row_cos(qs[l], -grads[l]) for l in range(len(qs))] + print("CHECK1 simultaneous n_dirs=32 cos(q,-g) per layer = " + + " ".join(f"{c:+.3f}" for c in cs)) + assert all(c > 0.2 for c in cs), "simultaneous perturbation should align in expectation" # ---- CHECK 2: SDIL overfits a fixed batch and alignment climbs ---- net = SDILNet(sizes, device=dev, seed=2) |