diff options
Diffstat (limited to 'collaborativeagents')
| -rw-r--r-- | collaborativeagents/adapters/personalized_llm_adapter.py | 34 | ||||
| -rwxr-xr-x | collaborativeagents/scripts/queue_ablation_zshort.sh | 47 | ||||
| -rw-r--r-- | collaborativeagents/scripts/run_experiments.py | 2 |
3 files changed, 83 insertions, 0 deletions
diff --git a/collaborativeagents/adapters/personalized_llm_adapter.py b/collaborativeagents/adapters/personalized_llm_adapter.py index 488b241..4189f31 100644 --- a/collaborativeagents/adapters/personalized_llm_adapter.py +++ b/collaborativeagents/adapters/personalized_llm_adapter.py @@ -672,6 +672,40 @@ def create_baseline_adapter( "extractor": "cuda:2", }, ), + # Ablation: RAG + Vector without z_short (only z_long, no within-session adaptation) + "rag_vector_no_short": AdapterConfig( + mode="full", + enable_preference_extraction=True, + enable_rl_updates=True, + use_user_vector=True, + llm_name=llm_name, + use_shared_models=use_shared_models, + enable_query_transform=True, + enable_global_preferences=True, + eta_short=0.0, # Disable z_short learning + device_assignment={ + "embed": "cuda:2", + "reranker": "cuda:3", + "extractor": "cuda:2", + }, + ), + # Ablation: RAG + Vector without z_long (only z_short, no cross-session learning) + "rag_vector_no_long": AdapterConfig( + mode="full", + enable_preference_extraction=True, + enable_rl_updates=True, + use_user_vector=True, + llm_name=llm_name, + use_shared_models=use_shared_models, + enable_query_transform=True, + enable_global_preferences=True, + eta_long=0.0, # Disable z_long learning + device_assignment={ + "embed": "cuda:2", + "reranker": "cuda:3", + "extractor": "cuda:2", + }, + ), # Baseline 7a: RAG + Vector + Preference Rewrite (combines best of both) "rag_rewrite_vector": AdapterConfig( mode="full", diff --git a/collaborativeagents/scripts/queue_ablation_zshort.sh b/collaborativeagents/scripts/queue_ablation_zshort.sh new file mode 100755 index 0000000..b4879c7 --- /dev/null +++ b/collaborativeagents/scripts/queue_ablation_zshort.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# Ablation experiment: z_short necessity +# Run rag_vector_no_short (z_long only) and rag_vector_no_long (z_short only) +# Same setup as 60s main experiment for direct comparison + +echo "Starting z_short ablation experiments..." +echo "$(date '+%Y-%m-%d %H:%M:%S')" + +# First: run rag_vector_no_short (only z_long, no within-session adaptation) +echo "Starting rag_vector_no_short..." +python collaborativeagents/scripts/run_experiments.py \ + --methods rag_vector_no_short \ + --datasets math-hard,math-500,bigcodebench \ + --n-profiles 60 \ + --n-sessions 60 \ + --max-turns 10 \ + --use-vllm \ + --vllm-agent-url http://localhost:8003/v1 \ + --vllm-user-url http://localhost:8004/v1 \ + --use-batch-processing \ + --batch-size 4 \ + --parallel-profiles 20 \ + --profile-path collaborativeagents/data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir collaborativeagents/results/ablation_no_short_60s \ + 2>&1 | tee collaborativeagents/results/ablation_no_short_60s.log + +echo "rag_vector_no_short completed at $(date '+%Y-%m-%d %H:%M:%S')" + +# Then: run rag_vector_no_long (only z_short, no cross-session learning) +echo "Starting rag_vector_no_long..." +python collaborativeagents/scripts/run_experiments.py \ + --methods rag_vector_no_long \ + --datasets math-hard,math-500,bigcodebench \ + --n-profiles 60 \ + --n-sessions 60 \ + --max-turns 10 \ + --use-vllm \ + --vllm-agent-url http://localhost:8003/v1 \ + --vllm-user-url http://localhost:8004/v1 \ + --use-batch-processing \ + --batch-size 4 \ + --parallel-profiles 20 \ + --profile-path collaborativeagents/data/complex_profiles_v2/profiles_200.jsonl \ + --output-dir collaborativeagents/results/ablation_no_long_60s \ + 2>&1 | tee collaborativeagents/results/ablation_no_long_60s.log + +echo "Both ablation experiments completed at $(date '+%Y-%m-%d %H:%M:%S')" diff --git a/collaborativeagents/scripts/run_experiments.py b/collaborativeagents/scripts/run_experiments.py index da3549b..806b959 100644 --- a/collaborativeagents/scripts/run_experiments.py +++ b/collaborativeagents/scripts/run_experiments.py @@ -118,6 +118,8 @@ AVAILABLE_METHODS = { "rag_rewrite": "Extractor + RAG with LLM preference rewrite/merge", "rag_rewrite_vector": "Extractor + RAG + user vector + LLM preference rewrite", "rag_vector": "Extractor + RAG + user vector (proposed method)", + "rag_vector_no_short": "Ablation: RAG + user vector without z_short (long-term only)", + "rag_vector_no_long": "Ablation: RAG + user vector without z_long (short-term only)", "rag_vector_fast": "Extractor + RAG + user vector with 10x learning rate", "rag_vector_consolidate": "Extractor + RAG + user vector with session-level preference consolidation", "rag_vector_balanced": "Extractor + RAG + user vector with balanced rewards (10x LR + positive signal for good turns)", |