From 107155532f2683198aac33c9bb3bd647d357a80a Mon Sep 17 00:00:00 2001 From: YurenHao0426 Date: Sun, 5 Apr 2026 18:10:38 -0500 Subject: Tune PromptTuning/PrefixTuning hyperparams: lr=1e-3/5e-4, steps=100 Previous lr=0.01 gave R-L=0.01 (broken output). Reduced to lr=1e-3 for PromptTuning, lr=5e-4 for PrefixTuning, increased steps from 30 to 100. Also made steps parameter configurable in _run_peft. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/run_all_methods.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/run_all_methods.py b/scripts/run_all_methods.py index db31a56..66b9cb7 100644 --- a/scripts/run_all_methods.py +++ b/scripts/run_all_methods.py @@ -100,11 +100,11 @@ class MethodRunner: 'lora': lambda *a: self._run_peft(*a, config=get_lora_config(rank=8), lr=1e-4, desc='LoRA r=8'), 'tiny_lora': lambda *a: self._run_peft(*a, config=get_tiny_lora_config(rank=1), lr=1e-4, desc='Tiny LoRA r=1'), 'vera': lambda *a: self._run_peft(*a, config=get_vera_config(rank=256), lr=1e-3, desc='VeRA r=256'), - 'prompt_tuning_5': lambda *a: self._run_peft(*a, config=get_prompt_tuning_config(5), lr=1e-2, desc='PromptTuning L=5'), - 'prompt_tuning_10': lambda *a: self._run_peft(*a, config=get_prompt_tuning_config(10), lr=1e-2, desc='PromptTuning L=10'), - 'prompt_tuning_20': lambda *a: self._run_peft(*a, config=get_prompt_tuning_config(20), lr=1e-2, desc='PromptTuning L=20'), - 'prefix_tuning_5': lambda *a: self._run_peft(*a, config=get_prefix_tuning_config(5), lr=1e-3, desc='PrefixTuning L=5'), - 'prefix_tuning_10': lambda *a: self._run_peft(*a, config=get_prefix_tuning_config(10), lr=1e-3, desc='PrefixTuning L=10'), + 'prompt_tuning_5': lambda *a: self._run_peft(*a, config=get_prompt_tuning_config(5), lr=1e-3, desc='PromptTuning L=5', steps=100), + 'prompt_tuning_10': lambda *a: self._run_peft(*a, config=get_prompt_tuning_config(10), lr=1e-3, desc='PromptTuning L=10', steps=100), + 'prompt_tuning_20': lambda *a: self._run_peft(*a, config=get_prompt_tuning_config(20), lr=1e-3, desc='PromptTuning L=20', steps=100), + 'prefix_tuning_5': lambda *a: self._run_peft(*a, config=get_prefix_tuning_config(5), lr=5e-4, desc='PrefixTuning L=5', steps=100), + 'prefix_tuning_10': lambda *a: self._run_peft(*a, config=get_prefix_tuning_config(10), lr=5e-4, desc='PrefixTuning L=10', steps=100), } if method_name not in dispatch: @@ -261,9 +261,9 @@ class MethodRunner: return per_user def _run_peft(self, examples, support_sets, references, support_texts, N, - config, lr, desc): + config, lr, desc, steps=30): baseline = PEFTBaseline(self.wrapper, config) - print(f" Trainable params: {baseline.n_params:,} ({baseline.n_bytes:,} bytes)") + print(f" Trainable params: {baseline.n_params:,} ({baseline.n_bytes:,} bytes), steps={steps}, lr={lr}") per_user = [] for i, (ex, support) in enumerate(zip(examples, support_sets)): @@ -272,7 +272,7 @@ class MethodRunner: support_items=support, query_input=ex['query_input'], task=ex['task'], - lr=lr, steps=30, + lr=lr, steps=steps, max_new_tokens=512, min_new_tokens=128, ) entry = self._make_per_user_entry( -- cgit v1.2.3